From 0799e8130f64afd2b4234a936d798af4365abe43 Mon Sep 17 00:00:00 2001
From: dojosout <dojosout@netsage-testing.tacc.utexas.edu>
Date: Wed, 31 Jul 2024 09:48:02 -0500
Subject: [PATCH] Deploy website - based on
 f69794d2ed9e72f49294818fb5be802bb297eb3c

---
 404.html                                      | 14 ++---
 assets/js/00cb704f.d8fb9eb4.js                |  1 -
 ...f72bc.fac18353.js => 012f72bc.36b7fe96.js} |  2 +-
 ...4960a.e4e7cd3e.js => 0214960a.7e6a0dc4.js} |  2 +-
 ...41ef2.5a4f1f57.js => 02341ef2.76f0a379.js} |  2 +-
 ...1802f.b404da72.js => 0271802f.e2ca0056.js} |  2 +-
 ...7c8bf.c122f958.js => 0327c8bf.4ed718d8.js} |  2 +-
 ...1afdd.426b41ab.js => 0381afdd.de4385a8.js} |  2 +-
 assets/js/05c6fd0e.27777f60.js                |  1 +
 assets/js/05c6fd0e.586b8e3b.js                |  1 -
 ...f8364.0e023e16.js => 066f8364.bfb615a4.js} |  2 +-
 ...addd5.24f5303f.js => 07faddd5.b2854b31.js} |  2 +-
 ...a21b4.34495c8a.js => 080a21b4.616746c2.js} |  2 +-
 assets/js/092c2d1d.97793caf.js                |  1 +
 ...e461a.4f5dc652.js => 0b8e461a.4a1a6155.js} |  2 +-
 assets/js/0d6d53ac.2ae3b199.js                |  1 -
 assets/js/0dd3dba6.3dbbc1f3.js                |  1 +
 assets/js/{1.bb3bc4d5.js => 1.03db48d6.js}    |  2 +-
 ...618b7.efff362b.js => 12ba72b5.4b8388c3.js} |  2 +-
 assets/js/12ecfc4a.a1b9aa60.js                |  1 +
 .../js/{161.d19b2b8e.js => 157.1f338186.js}   |  2 +-
 .../js/{162.71673972.js => 158.319f81c0.js}   |  2 +-
 ...cefd4.2fe48b23.js => 16fcefd4.9eb05a6f.js} |  2 +-
 ...96441.1b171940.js => 17896441.104e0abb.js} |  2 +-
 assets/js/1bbe794e.484f5992.js                |  1 -
 ...43674.8da06d41.js => 1cc43674.f9080c78.js} |  2 +-
 ...38b8a.35918ded.js => 1d038b8a.936f367a.js} |  2 +-
 assets/js/1f3a3353.117952bc.js                |  1 +
 assets/js/1f3a3353.c8d7370b.js                |  1 -
 assets/js/{2.89c52fba.js => 2.f7e8da3d.js}    |  4 +-
 ....LICENSE.txt => 2.f7e8da3d.js.LICENSE.txt} |  0
 ...e0bc4.17500718.js => 21ee0bc4.03721b9e.js} |  2 +-
 ...7278f.fe1cf5a5.js => 22b7278f.58242775.js} |  2 +-
 ...86c11.0650c59f.js => 22d86c11.2a981864.js} |  2 +-
 ...9f6dc.91efb769.js => 3179f6dc.bedc2974.js} |  2 +-
 ...9db24.62c09093.js => 3309db24.f6153687.js} |  2 +-
 ...6b1c9.bd4095bc.js => 33f6b1c9.798766ac.js} |  2 +-
 ...e6759.d9fbc6e9.js => 358e6759.19972cb0.js} |  2 +-
 assets/js/38962575.8edb151f.js                |  1 +
 assets/js/38962575.cec7a9ef.js                |  1 -
 ...1fe39.6ca6725c.js => 38d1fe39.e28e353a.js} |  2 +-
 ...4abfa.09b700f7.js => 3914abfa.df468ae2.js} |  2 +-
 ...960a4.2c1c192b.js => 40b960a4.422649da.js} |  2 +-
 ...80e0e.34ea6f7b.js => 41980e0e.d717c996.js} |  2 +-
 ...094bc.2568d8ec.js => 42e094bc.04cc4a66.js} |  2 +-
 ...0e773.55a7a6c7.js => 42f0e773.73889b7e.js} |  2 +-
 ...7ba93.8e66f0a3.js => 42f7ba93.e851b609.js} |  2 +-
 ...62f47.792b1adb.js => 43862f47.7a6ac57b.js} |  2 +-
 ...600e4.c1d369e6.js => 46a600e4.790fa3f6.js} |  2 +-
 ...c5911.4a2f1d5a.js => 480c5911.48d5baca.js} |  2 +-
 ...03371.0ed6dd81.js => 49703371.0286d91d.js} |  2 +-
 ...74cdd.e8f8d5a1.js => 49e74cdd.00e19770.js} |  2 +-
 ...4d8e8.964ee9cc.js => 4a74d8e8.d930cabd.js} |  2 +-
 ...24bf8.5f1f44d6.js => 4ff24bf8.c0099967.js} |  2 +-
 ...df56a.031ac316.js => 4ffdf56a.f556f662.js} |  2 +-
 ...32990.0ea61ccd.js => 50032990.47d0b7f0.js} |  2 +-
 ...d766a.b0ec51ab.js => 514d766a.ca35f79e.js} |  2 +-
 assets/js/53b0b5d8.144adf6f.js                |  1 -
 ...76a50.28321f6f.js => 56776a50.009ef339.js} |  2 +-
 ...29d1a.92eaa661.js => 56a29d1a.de0b8997.js} |  2 +-
 ...958f4.6a5669a4.js => 58d958f4.572280db.js} |  2 +-
 assets/js/59305d05.7e6bc501.js                |  1 -
 ...0f281.847d72f0.js => 5a20f281.84ff2815.js} |  2 +-
 ...c4cb7.5d8d3771.js => 5b6c4cb7.1e5700c2.js} |  2 +-
 ...beb71.a62d61d9.js => 5c4beb71.b666c6b3.js} |  2 +-
 ...17acf.eeb64f93.js => 5ee17acf.4e665490.js} |  2 +-
 assets/js/5ef24f1d.6c97f268.js                |  1 +
 ...5e2e5.da94766a.js => 6125e2e5.372c6163.js} |  2 +-
 assets/js/63503174.24f9a6f6.js                |  1 +
 ...5fe43.bed1e40c.js => 63f5fe43.faa7d00d.js} |  2 +-
 ...69ed8.20920114.js => 64269ed8.18f9818d.js} |  2 +-
 ...602f1.c1abe2fd.js => 653602f1.d686a47d.js} |  2 +-
 ...6ef57.e13c6102.js => 6546ef57.b480f0cc.js} |  2 +-
 assets/js/66e31e6e.ce62be57.js                |  1 -
 ...1008e.e9443d0d.js => 69c1008e.6660742f.js} |  2 +-
 ...59f5b.6c7d9748.js => 6a059f5b.57b36ba4.js} |  2 +-
 ...79c12.cc8d8222.js => 6c279c12.225c0b15.js} |  2 +-
 ...13b5d.84d394c4.js => 6d913b5d.bb807553.js} |  2 +-
 ...5514b.c223f8ef.js => 6e2a1c72.7c78a48b.js} |  2 +-
 assets/js/6e3166bc.fa40017d.js                |  1 +
 assets/js/71e5fd50.9e08f691.js                |  1 +
 assets/js/734dc415.80010e23.js                |  1 -
 assets/js/734dc415.9fadde42.js                |  1 +
 ...b1578.a774f21c.js => 799b1578.ad714f78.js} |  2 +-
 ...5b03c.95489b8b.js => 7b85b03c.6e3a64bd.js} |  2 +-
 ...8db09.bc8a300d.js => 7c78db09.5ea6e962.js} |  2 +-
 assets/js/7d0b7755.4b75443f.js                |  1 -
 assets/js/7d0b7755.c19774b1.js                |  1 +
 ...80c4c.ad3c0101.js => 7d380c4c.b92baa88.js} |  2 +-
 assets/js/8084fbd4.5592d29d.js                |  1 -
 ...ca04e.54bca9dc.js => 832ca04e.24044575.js} |  2 +-
 ...bc9a5.e629373c.js => 83dbc9a5.4a4e5652.js} |  2 +-
 assets/js/86d09e6e.43e743b3.js                |  1 +
 ...923b2.2bf3c135.js => 879923b2.90f3d9b4.js} |  2 +-
 ...7819f.64d4557b.js => 8997819f.aa7254b4.js} |  2 +-
 ...d3056.ff7822d9.js => 8b301107.d02fcb00.js} |  2 +-
 ...fcfa9.f0501de3.js => 8bbfcfa9.dc015f8e.js} |  2 +-
 assets/js/9001d127.1e177d17.js                |  1 -
 ...738b8.33482c0f.js => 906738b8.3a283baa.js} |  2 +-
 assets/js/9191c864.9b61c372.js                |  1 +
 assets/js/935f2afb.5c9a98e4.js                |  1 -
 assets/js/935f2afb.9f248d32.js                |  1 +
 ...52331.87e388ce.js => 96852331.822f904f.js} |  2 +-
 ...0d299.d4ac0286.js => 9720d299.9abe8149.js} |  2 +-
 ...dd56e.3b71c751.js => 97edd56e.d5ab66b5.js} |  2 +-
 ...819d4.508016b4.js => 992819d4.2993e926.js} |  2 +-
 ...6017e.6764251a.js => 99d6017e.cb54715a.js} |  2 +-
 ...c9747.5a3e990f.js => 9a5c9747.85ad9bca.js} |  2 +-
 assets/js/9aa54785.d3836c06.js                |  1 -
 ...05a1f.0d4ca405.js => 9b505a1f.f4b85bdc.js} |  2 +-
 ...dba8c.baa4761b.js => 9bfdba8c.b2391bbd.js} |  2 +-
 ...eac8e.4b2aa660.js => 9d5eac8e.6a3edbf9.js} |  2 +-
 assets/js/9e1ec80c.ab114838.js                |  1 -
 assets/js/9f628d52.08c37d03.js                |  1 +
 ...55709.25f4e00b.js => 9fe55709.e9c7bfda.js} |  2 +-
 ...af8ab.31cb6a73.js => 9feaf8ab.8c912915.js} |  2 +-
 ...bc459.cf90e3e8.js => a04bc459.dcff4f83.js} |  2 +-
 ...215ea.35e2776f.js => a12215ea.1ea3e4ef.js} |  2 +-
 assets/js/a3c17fac.4f9a964b.js                |  1 -
 assets/js/a55d7fc8.3675ba14.js                |  1 -
 assets/js/a55d7fc8.d2b40c2a.js                |  1 +
 ...59da2.03c88ff1.js => a7459da2.2d033572.js} |  2 +-
 ...8ec4a.ffbdd576.js => aa98ec4a.39b5137c.js} |  2 +-
 ...d28b2.21d293f7.js => aafd28b2.fb1f547f.js} |  2 +-
 ...f3a21.97ae394b.js => abbf3a21.2995f8fb.js} |  2 +-
 ...b7d69.fd3bbc4c.js => ac3b7d69.6126d7a7.js} |  2 +-
 ...08aff.9d4faea0.js => ad308aff.8a927dcc.js} |  2 +-
 ...a1364.d1aa581b.js => ae8a1364.4da8b56a.js} |  2 +-
 ...fe9f5.13251023.js => b1bfe9f5.592f8cc1.js} |  2 +-
 ...114f4.e41fb427.js => b3c114f4.28ba7bd1.js} |  2 +-
 assets/js/b56f03c6.3836580d.js                |  1 -
 ...a1049.1cf10416.js => b83a1049.f6464bc0.js} |  2 +-
 ...5e963.5bed7840.js => ba55e963.db919392.js} |  2 +-
 ...37b1e.9e57bcc8.js => bb237b1e.ea30ebd1.js} |  2 +-
 ...c6f88.e87a9bd6.js => bd6c6f88.8d988769.js} |  2 +-
 ...7ab4e.6b5e2485.js => bed7ab4e.fea77ca6.js} |  2 +-
 ...d86e8.4043049b.js => c0bd86e8.06ac168e.js} |  2 +-
 ...49368.7fcf56c0.js => c3f49368.e29a1244.js} |  2 +-
 ...5d8e4.76411a79.js => c4f5d8e4.4e0e3f0d.js} |  2 +-
 ...f4c72.0e8bec24.js => c51f4c72.69c062f8.js} |  2 +-
 assets/js/cbd1c0d0.5208c323.js                |  1 -
 assets/js/cbd1c0d0.6ea2feaf.js                |  1 +
 ...02920.a8ebc09f.js => cd902920.0ae4e688.js} |  2 +-
 assets/js/ceb45c47.a3330b5f.js                |  1 -
 assets/js/ceb45c47.e6acb003.js                |  1 +
 ...97237.705b0ef9.js => d1597237.07f1898e.js} |  2 +-
 assets/js/d1b2eded.fdea14ce.js                |  1 -
 assets/js/d2c255b4.51d1c434.js                |  1 -
 ...4f39e.b83f8cc2.js => d2c4f39e.d6c2c200.js} |  2 +-
 ...56f45.a168af5e.js => d3556f45.2aeec99a.js} |  2 +-
 ...b255e.d6a84f1e.js => d48b255e.1c5329df.js} |  2 +-
 ...87a67.4b6f0656.js => d5087a67.cb6a7230.js} |  2 +-
 ...40579.770e5402.js => d5140579.0f850372.js} |  2 +-
 ...79f42.d126a913.js => d5479f42.b8876961.js} |  2 +-
 ...ff009.7f914194.js => d71ff009.037347f4.js} |  2 +-
 assets/js/d74c151a.f23bb09f.js                |  1 +
 ...898ab.84f0767e.js => d75898ab.8089cc99.js} |  2 +-
 ...f9def.b4c78c77.js => d91f9def.f3256090.js} |  2 +-
 ...ed613.02dc005f.js => db0ed613.c9712562.js} |  2 +-
 ...a9f26.51ca4c1a.js => dd1a9f26.a545ed72.js} |  2 +-
 assets/js/ded423ee.df3ed6d9.js                |  1 -
 ...e17a0.481c92d6.js => e00e17a0.8c02aab3.js} |  2 +-
 ...970d8.6486e99c.js => e10970d8.cde1bc28.js} |  2 +-
 ...6625e.3f1e3436.js => e186625e.6c1b54b7.js} |  2 +-
 ...e698d.6ab0c41b.js => e2be698d.dd0e2d10.js} |  2 +-
 ...161d9.57d50314.js => e2c161d9.00dc4bde.js} |  2 +-
 ...bbbcf.e88d084c.js => e30bbbcf.b4e3431e.js} |  2 +-
 assets/js/e3649aa7.41406e34.js                |  1 -
 ...6ce66.ffd97d6e.js => e436ce66.54f16243.js} |  2 +-
 assets/js/e4c61278.03e947b6.js                |  1 -
 assets/js/e5ce8f83.b18dcb47.js                |  1 -
 ...6408e.8663b558.js => e746408e.d8730765.js} |  2 +-
 ...70743.b2234f2e.js => e7670743.fb9bbf42.js} |  2 +-
 ...7ec87.a4179a0d.js => e987ec87.f66b2f78.js} |  2 +-
 ...aece6.fbef59fb.js => ebfaece6.d97235a2.js} |  2 +-
 assets/js/f0b0282f.243b745a.js                |  1 +
 ...db0ed.0d7a9793.js => f1ddb0ed.4cbf6601.js} |  2 +-
 ...cfdf8.a481c25f.js => f2cce206.f146a973.js} |  2 +-
 ...5eca7.f3f2ea52.js => f485eca7.3a61264f.js} |  2 +-
 ...ba0ac.ba4c9a10.js => f52ba0ac.c7675cca.js} |  2 +-
 ...b3473.09cab9db.js => f64b3473.b96c3fd3.js} |  2 +-
 ...c0717.28c2f6ec.js => f8bc0717.e9c8f77e.js} |  2 +-
 assets/js/f9995f72.b29de3ff.js                |  1 +
 ...6f0fc.a192e853.js => fa16f0fc.8db267a7.js} |  2 +-
 assets/js/ff1d9e71.c2bc6db2.js                |  1 +
 assets/js/main.16ba1fad.js                    |  2 +
 ...CENSE.txt => main.16ba1fad.js.LICENSE.txt} |  0
 assets/js/main.e20b6a1c.js                    |  2 -
 assets/js/runtime~main.09475183.js            |  1 +
 assets/js/runtime~main.6b5de359.js            |  1 -
 ...{styles.19dac7a3.js => styles.ed63c98d.js} |  2 +-
 docs/1.2.10/components/docker_env/index.html  | 44 +++++++--------
 .../components/docker_first_steps/index.html  | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../deploy/bare_metal_install/index.html      | 44 +++++++--------
 docs/1.2.10/deploy/choose_install/index.html  | 42 +++++++-------
 .../deploy/docker_install_advanced/index.html | 44 +++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/1.2.10/deploy/docker_upgrade/index.html  | 44 +++++++--------
 docs/1.2.10/devel/dev_dataset/index.html      | 44 +++++++--------
 docs/1.2.10/devel/docker_dev_guide/index.html | 44 +++++++--------
 docs/1.2.10/devel/docker_dev_tag/index.html   | 44 +++++++--------
 docs/1.2.10/devel/docusaurus/index.html       | 44 +++++++--------
 docs/1.2.10/pipeline/elastic/index.html       | 42 +++++++-------
 docs/1.2.10/pipeline/importer/index.html      | 44 +++++++--------
 docs/1.2.10/pipeline/intro/index.html         | 44 +++++++--------
 docs/1.2.10/pipeline/logstash/index.html      | 44 +++++++--------
 docs/1.2.10/pipeline/nfdump/index.html        | 42 +++++++-------
 docs/1.2.10/pipeline/tstat/index.html         | 42 +++++++-------
 docs/1.2.11/components/docker_env/index.html  | 44 +++++++--------
 .../components/docker_first_steps/index.html  | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../deploy/bare_metal_install/index.html      | 44 +++++++--------
 docs/1.2.11/deploy/choose_install/index.html  | 42 +++++++-------
 .../deploy/docker_install_advanced/index.html | 44 +++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/1.2.11/deploy/docker_upgrade/index.html  | 42 +++++++-------
 docs/1.2.11/devel/dev_dataset/index.html      | 44 +++++++--------
 docs/1.2.11/devel/docker_dev_guide/index.html | 44 +++++++--------
 docs/1.2.11/devel/docker_dev_tag/index.html   | 44 +++++++--------
 docs/1.2.11/devel/docusaurus/index.html       | 44 +++++++--------
 docs/1.2.11/pipeline/elastic/index.html       | 42 +++++++-------
 docs/1.2.11/pipeline/importer/index.html      | 44 +++++++--------
 docs/1.2.11/pipeline/intro/index.html         | 44 +++++++--------
 docs/1.2.11/pipeline/logstash/index.html      | 44 +++++++--------
 docs/1.2.11/pipeline/nfdump/index.html        | 42 +++++++-------
 docs/1.2.11/pipeline/tstat/index.html         | 42 +++++++-------
 docs/1.2.5/deploy/choose_install/index.html   | 42 +++++++-------
 docs/1.2.5/deploy/docker_install/index.html   | 44 +++++++--------
 docs/1.2.5/deploy/install/index.html          | 44 +++++++--------
 docs/1.2.5/devel/docker/index.html            | 44 +++++++--------
 docs/1.2.5/docusaurus/index.html              | 44 +++++++--------
 docs/1.2.5/pipeline/index.html                | 42 +++++++-------
 docs/1.2.5/pipeline_importer/index.html       | 44 +++++++--------
 docs/1.2.5/pipeline_logstash/index.html       | 44 +++++++--------
 docs/1.2.6/deploy/choose_install/index.html   | 42 +++++++-------
 docs/1.2.6/deploy/docker_install/index.html   | 44 +++++++--------
 docs/1.2.6/deploy/install/index.html          | 44 +++++++--------
 docs/1.2.6/devel/docker/index.html            | 44 +++++++--------
 docs/1.2.6/docusaurus/index.html              | 44 +++++++--------
 docs/1.2.6/pipeline/index.html                | 42 +++++++-------
 docs/1.2.6/pipeline_importer/index.html       | 44 +++++++--------
 docs/1.2.6/pipeline_logstash/index.html       | 44 +++++++--------
 docs/1.2.7/components/docker_env/index.html   | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../components/docker_upgrade/index.html      | 42 +++++++-------
 .../deploy/bare_metal_install/index.html      | 44 +++++++--------
 docs/1.2.7/deploy/choose_install/index.html   | 42 +++++++-------
 .../deploy/docker_install_advanced/index.html | 44 +++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/1.2.7/devel/dev_dataset/index.html       | 44 +++++++--------
 docs/1.2.7/devel/docker_dev_guide/index.html  | 44 +++++++--------
 docs/1.2.7/docusaurus/index.html              | 44 +++++++--------
 docs/1.2.7/pipeline/elastic/index.html        | 42 +++++++-------
 docs/1.2.7/pipeline/importer/index.html       | 44 +++++++--------
 docs/1.2.7/pipeline/intro/index.html          | 44 +++++++--------
 docs/1.2.7/pipeline/logstash/index.html       | 44 +++++++--------
 docs/1.2.7/pipeline/nfdump/index.html         | 42 +++++++-------
 docs/1.2.7/pipeline/tstat/index.html          | 42 +++++++-------
 docs/1.2.8/components/docker_env/index.html   | 44 +++++++--------
 .../components/docker_first_steps/index.html  | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../components/docker_upgrade/index.html      | 44 +++++++--------
 .../deploy/bare_metal_install/index.html      | 44 +++++++--------
 docs/1.2.8/deploy/choose_install/index.html   | 42 +++++++-------
 .../deploy/docker_install_advanced/index.html | 44 +++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/1.2.8/devel/dev_dataset/index.html       | 44 +++++++--------
 docs/1.2.8/devel/docker_dev_guide/index.html  | 44 +++++++--------
 docs/1.2.8/docusaurus/index.html              | 44 +++++++--------
 docs/1.2.8/pipeline/elastic/index.html        | 42 +++++++-------
 docs/1.2.8/pipeline/importer/index.html       | 44 +++++++--------
 docs/1.2.8/pipeline/intro/index.html          | 44 +++++++--------
 docs/1.2.8/pipeline/logstash/index.html       | 44 +++++++--------
 docs/1.2.8/pipeline/nfdump/index.html         | 42 +++++++-------
 docs/1.2.8/pipeline/tstat/index.html          | 42 +++++++-------
 docs/1.2.9/components/docker_env/index.html   | 44 +++++++--------
 .../components/docker_first_steps/index.html  | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../components/docker_upgrade/index.html      | 44 +++++++--------
 .../deploy/bare_metal_install/index.html      | 44 +++++++--------
 docs/1.2.9/deploy/choose_install/index.html   | 42 +++++++-------
 .../deploy/docker_install_advanced/index.html | 44 +++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/1.2.9/devel/dev_dataset/index.html       | 44 +++++++--------
 docs/1.2.9/devel/docker_dev_guide/index.html  | 44 +++++++--------
 docs/1.2.9/docusaurus/index.html              | 44 +++++++--------
 docs/1.2.9/pipeline/elastic/index.html        | 42 +++++++-------
 docs/1.2.9/pipeline/importer/index.html       | 44 +++++++--------
 docs/1.2.9/pipeline/intro/index.html          | 44 +++++++--------
 docs/1.2.9/pipeline/logstash/index.html       | 44 +++++++--------
 docs/1.2.9/pipeline/nfdump/index.html         | 42 +++++++-------
 docs/1.2.9/pipeline/tstat/index.html          | 42 +++++++-------
 docs/components/docker_env/index.html         | 46 +++++++--------
 docs/components/docker_first_steps/index.html | 46 +++++++--------
 docs/components/docker_pipeline/index.html    | 44 +++++++--------
 docs/deploy/bare_metal_install/index.html     | 56 -------------------
 docs/deploy/choose_install/index.html         | 34 -----------
 .../deploy/docker_install_advanced/index.html | 49 ++++++++--------
 docs/deploy/docker_install_simple/index.html  | 46 +++++++--------
 docs/deploy/docker_troubleshoot/index.html    | 46 +++++++--------
 docs/deploy/docker_upgrade/index.html         | 44 +++++++--------
 docs/devel/dev_dataset/index.html             | 46 +++++++--------
 docs/devel/docker_dev_guide/index.html        | 47 ++++++++--------
 docs/devel/docker_dev_tag/index.html          | 46 +++++++--------
 docs/devel/docusaurus/index.html              | 46 +++++++--------
 docs/next/components/docker_env/index.html    | 44 +++++++--------
 .../components/docker_first_steps/index.html  | 44 +++++++--------
 .../components/docker_pipeline/index.html     | 42 +++++++-------
 .../next/deploy/bare_metal_install/index.html | 56 -------------------
 docs/next/deploy/choose_install/index.html    | 34 -----------
 .../deploy/docker_install_advanced/index.html | 47 ++++++++--------
 .../deploy/docker_install_simple/index.html   | 44 +++++++--------
 .../deploy/docker_troubleshoot/index.html     | 44 +++++++--------
 docs/next/deploy/docker_upgrade/index.html    | 42 +++++++-------
 docs/next/devel/dev_dataset/index.html        | 44 +++++++--------
 docs/next/devel/docker_dev_guide/index.html   | 45 ++++++++-------
 docs/next/devel/docker_dev_tag/index.html     | 44 +++++++--------
 docs/next/devel/docusaurus/index.html         | 44 +++++++--------
 docs/next/pipeline/elastic/index.html         | 42 +++++++-------
 docs/next/pipeline/importer/index.html        | 44 +++++++--------
 docs/next/pipeline/intro/index.html           | 44 +++++++--------
 docs/next/pipeline/logstash/index.html        | 44 +++++++--------
 docs/next/pipeline/nfdump/index.html          | 42 +++++++-------
 docs/next/pipeline/tstat/index.html           | 42 +++++++-------
 docs/pipeline/elastic/index.html              | 44 +++++++--------
 docs/pipeline/importer/index.html             | 46 +++++++--------
 docs/pipeline/intro/index.html                | 46 +++++++--------
 docs/pipeline/logstash/index.html             | 46 +++++++--------
 docs/pipeline/nfdump/index.html               | 44 +++++++--------
 docs/pipeline/tstat/index.html                | 44 +++++++--------
 img/favicon.ico:Zone.Identifier               |  3 -
 img/logo.png:Zone.Identifier                  |  3 -
 img/logo.svg:Zone.Identifier                  |  3 -
 img/netsage.webp:Zone.Identifier              |  3 -
 img/netsage_bg.webp:Zone.Identifier           |  3 -
 ...aw_docusaurus_mountain.svg:Zone.Identifier |  3 -
 ...ndraw_docusaurus_react.svg:Zone.Identifier |  3 -
 ...undraw_docusaurus_tree.svg:Zone.Identifier |  3 -
 index.html                                    | 26 ++++-----
 sitemap.xml                                   |  2 +-
 346 files changed, 3262 insertions(+), 3474 deletions(-)
 delete mode 100644 assets/js/00cb704f.d8fb9eb4.js
 rename assets/js/{012f72bc.fac18353.js => 012f72bc.36b7fe96.js} (98%)
 rename assets/js/{0214960a.e4e7cd3e.js => 0214960a.7e6a0dc4.js} (98%)
 rename assets/js/{02341ef2.5a4f1f57.js => 02341ef2.76f0a379.js} (98%)
 rename assets/js/{0271802f.b404da72.js => 0271802f.e2ca0056.js} (98%)
 rename assets/js/{0327c8bf.c122f958.js => 0327c8bf.4ed718d8.js} (99%)
 rename assets/js/{0381afdd.426b41ab.js => 0381afdd.de4385a8.js} (97%)
 create mode 100644 assets/js/05c6fd0e.27777f60.js
 delete mode 100644 assets/js/05c6fd0e.586b8e3b.js
 rename assets/js/{066f8364.0e023e16.js => 066f8364.bfb615a4.js} (96%)
 rename assets/js/{07faddd5.24f5303f.js => 07faddd5.b2854b31.js} (97%)
 rename assets/js/{080a21b4.34495c8a.js => 080a21b4.616746c2.js} (96%)
 create mode 100644 assets/js/092c2d1d.97793caf.js
 rename assets/js/{0b8e461a.4f5dc652.js => 0b8e461a.4a1a6155.js} (97%)
 delete mode 100644 assets/js/0d6d53ac.2ae3b199.js
 create mode 100644 assets/js/0dd3dba6.3dbbc1f3.js
 rename assets/js/{1.bb3bc4d5.js => 1.03db48d6.js} (84%)
 rename assets/js/{418618b7.efff362b.js => 12ba72b5.4b8388c3.js} (84%)
 create mode 100644 assets/js/12ecfc4a.a1b9aa60.js
 rename assets/js/{161.d19b2b8e.js => 157.1f338186.js} (95%)
 rename assets/js/{162.71673972.js => 158.319f81c0.js} (80%)
 rename assets/js/{16fcefd4.2fe48b23.js => 16fcefd4.9eb05a6f.js} (98%)
 rename assets/js/{17896441.1b171940.js => 17896441.104e0abb.js} (92%)
 delete mode 100644 assets/js/1bbe794e.484f5992.js
 rename assets/js/{1cc43674.8da06d41.js => 1cc43674.f9080c78.js} (96%)
 rename assets/js/{1d038b8a.35918ded.js => 1d038b8a.936f367a.js} (97%)
 create mode 100644 assets/js/1f3a3353.117952bc.js
 delete mode 100644 assets/js/1f3a3353.c8d7370b.js
 rename assets/js/{2.89c52fba.js => 2.f7e8da3d.js} (90%)
 rename assets/js/{2.89c52fba.js.LICENSE.txt => 2.f7e8da3d.js.LICENSE.txt} (100%)
 rename assets/js/{21ee0bc4.17500718.js => 21ee0bc4.03721b9e.js} (96%)
 rename assets/js/{22b7278f.fe1cf5a5.js => 22b7278f.58242775.js} (97%)
 rename assets/js/{22d86c11.0650c59f.js => 22d86c11.2a981864.js} (99%)
 rename assets/js/{3179f6dc.91efb769.js => 3179f6dc.bedc2974.js} (96%)
 rename assets/js/{3309db24.62c09093.js => 3309db24.f6153687.js} (99%)
 rename assets/js/{33f6b1c9.bd4095bc.js => 33f6b1c9.798766ac.js} (97%)
 rename assets/js/{358e6759.d9fbc6e9.js => 358e6759.19972cb0.js} (97%)
 create mode 100644 assets/js/38962575.8edb151f.js
 delete mode 100644 assets/js/38962575.cec7a9ef.js
 rename assets/js/{38d1fe39.6ca6725c.js => 38d1fe39.e28e353a.js} (99%)
 rename assets/js/{3914abfa.09b700f7.js => 3914abfa.df468ae2.js} (96%)
 rename assets/js/{40b960a4.2c1c192b.js => 40b960a4.422649da.js} (96%)
 rename assets/js/{41980e0e.34ea6f7b.js => 41980e0e.d717c996.js} (98%)
 rename assets/js/{42e094bc.2568d8ec.js => 42e094bc.04cc4a66.js} (99%)
 rename assets/js/{42f0e773.55a7a6c7.js => 42f0e773.73889b7e.js} (97%)
 rename assets/js/{42f7ba93.8e66f0a3.js => 42f7ba93.e851b609.js} (97%)
 rename assets/js/{43862f47.792b1adb.js => 43862f47.7a6ac57b.js} (97%)
 rename assets/js/{46a600e4.c1d369e6.js => 46a600e4.790fa3f6.js} (97%)
 rename assets/js/{480c5911.4a2f1d5a.js => 480c5911.48d5baca.js} (97%)
 rename assets/js/{49703371.0ed6dd81.js => 49703371.0286d91d.js} (99%)
 rename assets/js/{49e74cdd.e8f8d5a1.js => 49e74cdd.00e19770.js} (98%)
 rename assets/js/{4a74d8e8.964ee9cc.js => 4a74d8e8.d930cabd.js} (99%)
 rename assets/js/{4ff24bf8.5f1f44d6.js => 4ff24bf8.c0099967.js} (97%)
 rename assets/js/{4ffdf56a.031ac316.js => 4ffdf56a.f556f662.js} (98%)
 rename assets/js/{50032990.0ea61ccd.js => 50032990.47d0b7f0.js} (97%)
 rename assets/js/{514d766a.b0ec51ab.js => 514d766a.ca35f79e.js} (98%)
 delete mode 100644 assets/js/53b0b5d8.144adf6f.js
 rename assets/js/{56776a50.28321f6f.js => 56776a50.009ef339.js} (98%)
 rename assets/js/{56a29d1a.92eaa661.js => 56a29d1a.de0b8997.js} (99%)
 rename assets/js/{58d958f4.6a5669a4.js => 58d958f4.572280db.js} (97%)
 delete mode 100644 assets/js/59305d05.7e6bc501.js
 rename assets/js/{5a20f281.847d72f0.js => 5a20f281.84ff2815.js} (99%)
 rename assets/js/{5b6c4cb7.5d8d3771.js => 5b6c4cb7.1e5700c2.js} (99%)
 rename assets/js/{5c4beb71.a62d61d9.js => 5c4beb71.b666c6b3.js} (96%)
 rename assets/js/{5ee17acf.eeb64f93.js => 5ee17acf.4e665490.js} (99%)
 create mode 100644 assets/js/5ef24f1d.6c97f268.js
 rename assets/js/{6125e2e5.da94766a.js => 6125e2e5.372c6163.js} (97%)
 create mode 100644 assets/js/63503174.24f9a6f6.js
 rename assets/js/{63f5fe43.bed1e40c.js => 63f5fe43.faa7d00d.js} (98%)
 rename assets/js/{64269ed8.20920114.js => 64269ed8.18f9818d.js} (98%)
 rename assets/js/{653602f1.c1abe2fd.js => 653602f1.d686a47d.js} (98%)
 rename assets/js/{6546ef57.e13c6102.js => 6546ef57.b480f0cc.js} (98%)
 delete mode 100644 assets/js/66e31e6e.ce62be57.js
 rename assets/js/{69c1008e.e9443d0d.js => 69c1008e.6660742f.js} (95%)
 rename assets/js/{6a059f5b.6c7d9748.js => 6a059f5b.57b36ba4.js} (98%)
 rename assets/js/{6c279c12.cc8d8222.js => 6c279c12.225c0b15.js} (96%)
 rename assets/js/{6d913b5d.84d394c4.js => 6d913b5d.bb807553.js} (98%)
 rename assets/js/{89d5514b.c223f8ef.js => 6e2a1c72.7c78a48b.js} (91%)
 create mode 100644 assets/js/6e3166bc.fa40017d.js
 create mode 100644 assets/js/71e5fd50.9e08f691.js
 delete mode 100644 assets/js/734dc415.80010e23.js
 create mode 100644 assets/js/734dc415.9fadde42.js
 rename assets/js/{799b1578.a774f21c.js => 799b1578.ad714f78.js} (96%)
 rename assets/js/{7b85b03c.95489b8b.js => 7b85b03c.6e3a64bd.js} (96%)
 rename assets/js/{7c78db09.bc8a300d.js => 7c78db09.5ea6e962.js} (97%)
 delete mode 100644 assets/js/7d0b7755.4b75443f.js
 create mode 100644 assets/js/7d0b7755.c19774b1.js
 rename assets/js/{7d380c4c.ad3c0101.js => 7d380c4c.b92baa88.js} (96%)
 delete mode 100644 assets/js/8084fbd4.5592d29d.js
 rename assets/js/{832ca04e.54bca9dc.js => 832ca04e.24044575.js} (95%)
 rename assets/js/{83dbc9a5.e629373c.js => 83dbc9a5.4a4e5652.js} (98%)
 create mode 100644 assets/js/86d09e6e.43e743b3.js
 rename assets/js/{879923b2.2bf3c135.js => 879923b2.90f3d9b4.js} (99%)
 rename assets/js/{8997819f.64d4557b.js => 8997819f.aa7254b4.js} (97%)
 rename assets/js/{9b6d3056.ff7822d9.js => 8b301107.d02fcb00.js} (95%)
 rename assets/js/{8bbfcfa9.f0501de3.js => 8bbfcfa9.dc015f8e.js} (96%)
 delete mode 100644 assets/js/9001d127.1e177d17.js
 rename assets/js/{906738b8.33482c0f.js => 906738b8.3a283baa.js} (97%)
 create mode 100644 assets/js/9191c864.9b61c372.js
 delete mode 100644 assets/js/935f2afb.5c9a98e4.js
 create mode 100644 assets/js/935f2afb.9f248d32.js
 rename assets/js/{96852331.87e388ce.js => 96852331.822f904f.js} (96%)
 rename assets/js/{9720d299.d4ac0286.js => 9720d299.9abe8149.js} (98%)
 rename assets/js/{97edd56e.3b71c751.js => 97edd56e.d5ab66b5.js} (98%)
 rename assets/js/{992819d4.508016b4.js => 992819d4.2993e926.js} (97%)
 rename assets/js/{99d6017e.6764251a.js => 99d6017e.cb54715a.js} (96%)
 rename assets/js/{9a5c9747.5a3e990f.js => 9a5c9747.85ad9bca.js} (96%)
 delete mode 100644 assets/js/9aa54785.d3836c06.js
 rename assets/js/{9b505a1f.0d4ca405.js => 9b505a1f.f4b85bdc.js} (98%)
 rename assets/js/{9bfdba8c.baa4761b.js => 9bfdba8c.b2391bbd.js} (97%)
 rename assets/js/{9d5eac8e.4b2aa660.js => 9d5eac8e.6a3edbf9.js} (98%)
 delete mode 100644 assets/js/9e1ec80c.ab114838.js
 create mode 100644 assets/js/9f628d52.08c37d03.js
 rename assets/js/{9fe55709.25f4e00b.js => 9fe55709.e9c7bfda.js} (99%)
 rename assets/js/{9feaf8ab.31cb6a73.js => 9feaf8ab.8c912915.js} (99%)
 rename assets/js/{a04bc459.cf90e3e8.js => a04bc459.dcff4f83.js} (98%)
 rename assets/js/{a12215ea.35e2776f.js => a12215ea.1ea3e4ef.js} (98%)
 delete mode 100644 assets/js/a3c17fac.4f9a964b.js
 delete mode 100644 assets/js/a55d7fc8.3675ba14.js
 create mode 100644 assets/js/a55d7fc8.d2b40c2a.js
 rename assets/js/{a7459da2.03c88ff1.js => a7459da2.2d033572.js} (99%)
 rename assets/js/{aa98ec4a.ffbdd576.js => aa98ec4a.39b5137c.js} (98%)
 rename assets/js/{aafd28b2.21d293f7.js => aafd28b2.fb1f547f.js} (99%)
 rename assets/js/{abbf3a21.97ae394b.js => abbf3a21.2995f8fb.js} (97%)
 rename assets/js/{ac3b7d69.fd3bbc4c.js => ac3b7d69.6126d7a7.js} (98%)
 rename assets/js/{ad308aff.9d4faea0.js => ad308aff.8a927dcc.js} (96%)
 rename assets/js/{ae8a1364.d1aa581b.js => ae8a1364.4da8b56a.js} (96%)
 rename assets/js/{b1bfe9f5.13251023.js => b1bfe9f5.592f8cc1.js} (96%)
 rename assets/js/{b3c114f4.e41fb427.js => b3c114f4.28ba7bd1.js} (96%)
 delete mode 100644 assets/js/b56f03c6.3836580d.js
 rename assets/js/{b83a1049.1cf10416.js => b83a1049.f6464bc0.js} (96%)
 rename assets/js/{ba55e963.5bed7840.js => ba55e963.db919392.js} (95%)
 rename assets/js/{bb237b1e.9e57bcc8.js => bb237b1e.ea30ebd1.js} (98%)
 rename assets/js/{bd6c6f88.e87a9bd6.js => bd6c6f88.8d988769.js} (98%)
 rename assets/js/{bed7ab4e.6b5e2485.js => bed7ab4e.fea77ca6.js} (99%)
 rename assets/js/{c0bd86e8.4043049b.js => c0bd86e8.06ac168e.js} (96%)
 rename assets/js/{c3f49368.7fcf56c0.js => c3f49368.e29a1244.js} (97%)
 rename assets/js/{c4f5d8e4.76411a79.js => c4f5d8e4.4e0e3f0d.js} (78%)
 rename assets/js/{c51f4c72.0e8bec24.js => c51f4c72.69c062f8.js} (99%)
 delete mode 100644 assets/js/cbd1c0d0.5208c323.js
 create mode 100644 assets/js/cbd1c0d0.6ea2feaf.js
 rename assets/js/{cd902920.a8ebc09f.js => cd902920.0ae4e688.js} (97%)
 delete mode 100644 assets/js/ceb45c47.a3330b5f.js
 create mode 100644 assets/js/ceb45c47.e6acb003.js
 rename assets/js/{d1597237.705b0ef9.js => d1597237.07f1898e.js} (95%)
 delete mode 100644 assets/js/d1b2eded.fdea14ce.js
 delete mode 100644 assets/js/d2c255b4.51d1c434.js
 rename assets/js/{d2c4f39e.b83f8cc2.js => d2c4f39e.d6c2c200.js} (96%)
 rename assets/js/{d3556f45.a168af5e.js => d3556f45.2aeec99a.js} (95%)
 rename assets/js/{d48b255e.d6a84f1e.js => d48b255e.1c5329df.js} (98%)
 rename assets/js/{d5087a67.4b6f0656.js => d5087a67.cb6a7230.js} (98%)
 rename assets/js/{d5140579.770e5402.js => d5140579.0f850372.js} (97%)
 rename assets/js/{d5479f42.d126a913.js => d5479f42.b8876961.js} (96%)
 rename assets/js/{d71ff009.7f914194.js => d71ff009.037347f4.js} (96%)
 create mode 100644 assets/js/d74c151a.f23bb09f.js
 rename assets/js/{d75898ab.84f0767e.js => d75898ab.8089cc99.js} (97%)
 rename assets/js/{d91f9def.b4c78c77.js => d91f9def.f3256090.js} (95%)
 rename assets/js/{db0ed613.02dc005f.js => db0ed613.c9712562.js} (97%)
 rename assets/js/{dd1a9f26.51ca4c1a.js => dd1a9f26.a545ed72.js} (99%)
 delete mode 100644 assets/js/ded423ee.df3ed6d9.js
 rename assets/js/{e00e17a0.481c92d6.js => e00e17a0.8c02aab3.js} (96%)
 rename assets/js/{e10970d8.6486e99c.js => e10970d8.cde1bc28.js} (97%)
 rename assets/js/{e186625e.3f1e3436.js => e186625e.6c1b54b7.js} (95%)
 rename assets/js/{e2be698d.6ab0c41b.js => e2be698d.dd0e2d10.js} (99%)
 rename assets/js/{e2c161d9.57d50314.js => e2c161d9.00dc4bde.js} (95%)
 rename assets/js/{e30bbbcf.e88d084c.js => e30bbbcf.b4e3431e.js} (96%)
 delete mode 100644 assets/js/e3649aa7.41406e34.js
 rename assets/js/{e436ce66.ffd97d6e.js => e436ce66.54f16243.js} (94%)
 delete mode 100644 assets/js/e4c61278.03e947b6.js
 delete mode 100644 assets/js/e5ce8f83.b18dcb47.js
 rename assets/js/{e746408e.8663b558.js => e746408e.d8730765.js} (99%)
 rename assets/js/{e7670743.b2234f2e.js => e7670743.fb9bbf42.js} (99%)
 rename assets/js/{e987ec87.a4179a0d.js => e987ec87.f66b2f78.js} (96%)
 rename assets/js/{ebfaece6.fbef59fb.js => ebfaece6.d97235a2.js} (96%)
 create mode 100644 assets/js/f0b0282f.243b745a.js
 rename assets/js/{f1ddb0ed.0d7a9793.js => f1ddb0ed.4cbf6601.js} (99%)
 rename assets/js/{af9cfdf8.a481c25f.js => f2cce206.f146a973.js} (94%)
 rename assets/js/{f485eca7.f3f2ea52.js => f485eca7.3a61264f.js} (96%)
 rename assets/js/{f52ba0ac.ba4c9a10.js => f52ba0ac.c7675cca.js} (95%)
 rename assets/js/{f64b3473.09cab9db.js => f64b3473.b96c3fd3.js} (96%)
 rename assets/js/{f8bc0717.28c2f6ec.js => f8bc0717.e9c8f77e.js} (96%)
 create mode 100644 assets/js/f9995f72.b29de3ff.js
 rename assets/js/{fa16f0fc.a192e853.js => fa16f0fc.8db267a7.js} (96%)
 create mode 100644 assets/js/ff1d9e71.c2bc6db2.js
 create mode 100644 assets/js/main.16ba1fad.js
 rename assets/js/{main.e20b6a1c.js.LICENSE.txt => main.16ba1fad.js.LICENSE.txt} (100%)
 delete mode 100644 assets/js/main.e20b6a1c.js
 create mode 100644 assets/js/runtime~main.09475183.js
 delete mode 100644 assets/js/runtime~main.6b5de359.js
 rename assets/js/{styles.19dac7a3.js => styles.ed63c98d.js} (77%)
 delete mode 100644 docs/deploy/bare_metal_install/index.html
 delete mode 100644 docs/deploy/choose_install/index.html
 delete mode 100644 docs/next/deploy/bare_metal_install/index.html
 delete mode 100644 docs/next/deploy/choose_install/index.html
 delete mode 100644 img/favicon.ico:Zone.Identifier
 delete mode 100644 img/logo.png:Zone.Identifier
 delete mode 100644 img/logo.svg:Zone.Identifier
 delete mode 100644 img/netsage.webp:Zone.Identifier
 delete mode 100644 img/netsage_bg.webp:Zone.Identifier
 delete mode 100644 img/undraw_docusaurus_mountain.svg:Zone.Identifier
 delete mode 100644 img/undraw_docusaurus_react.svg:Zone.Identifier
 delete mode 100644 img/undraw_docusaurus_tree.svg:Zone.Identifier

diff --git a/404.html b/404.html
index 884da7bf..f8915627 100644
--- a/404.html
+++ b/404.html
@@ -6,15 +6,15 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Page Not Found | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:title" content="Page Not Found | Netsage Pipeline Documentation"><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/404.html"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_tag" content="default"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/404.html"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/404.html" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/404.html" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper"><main class="container margin-vert--xl"><div class="row"><div class="col col--6 col--offset-3"><h1 class="hero__title">Page Not Found</h1><p>We could not find what you were looking for.</p><p>Please contact the owner of the site that linked you to the original URL and let them know their link is broken.</p></div></div></main></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper"><main class="container margin-vert--xl"><div class="row"><div class="col col--6 col--offset-3"><h1 class="hero__title">Page Not Found</h1><p>We could not find what you were looking for.</p><p>Please contact the owner of the site that linked you to the original URL and let them know their link is broken.</p></div></div></main></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/assets/js/00cb704f.d8fb9eb4.js b/assets/js/00cb704f.d8fb9eb4.js
deleted file mode 100644
index 67ac71dc..00000000
--- a/assets/js/00cb704f.d8fb9eb4.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[3],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(i,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,l(l({ref:t},c),{},{components:n})):a.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},65:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-2.0.0/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-2.0.0/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_install_advanced.md",version:"2.0.0",sidebar_label:"Docker Advanced Options",sidebar:"version-2.0.0/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Filter Flows by Subnet",id:"to-filter-flows-by-subnet",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-filter-flows-by-subnet"},"To Filter Flows by Subnet"),Object(r.b)("p",null,'With this option, flows from specified sensors will be dropped unless src or dst is in the list of subnets to keep. It works similarly to the option to filter by interface.  "ALL" can refer to all sensors.\nIf a sensor is not referenced at all, all of its flows will be kept. '),Object(r.b)("p",null,"For example,"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"subnet_filter_flag=True\nsubnet_filter_keep=Sensor A Name: 123.45.6.0/16; Sensor B Name: 123.33.33.0/24, 456.66.66.0/24\n")),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/012f72bc.fac18353.js b/assets/js/012f72bc.36b7fe96.js
similarity index 98%
rename from assets/js/012f72bc.fac18353.js
rename to assets/js/012f72bc.36b7fe96.js
index 225d8cda..704c4a00 100644
--- a/assets/js/012f72bc.fac18353.js
+++ b/assets/js/012f72bc.36b7fe96.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[4],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},66:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.11/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-1.2.11/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/logstash.md",version:"1.2.11",sidebar_label:"Logstash",sidebar:"version-1.2.11/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[3],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},65:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.11/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-1.2.11/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/logstash.md",version:"1.2.11",sidebar_label:"Logstash",sidebar:"version-1.2.11/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0214960a.e4e7cd3e.js b/assets/js/0214960a.7e6a0dc4.js
similarity index 98%
rename from assets/js/0214960a.e4e7cd3e.js
rename to assets/js/0214960a.7e6a0dc4.js
index f6602007..55da6741 100644
--- a/assets/js/0214960a.e4e7cd3e.js
+++ b/assets/js/0214960a.7e6a0dc4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[5],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},67:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.8/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.8/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/docker.md",version:"1.2.8",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.8/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[4],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},66:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.8/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.8/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/docker.md",version:"1.2.8",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.8/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/02341ef2.5a4f1f57.js b/assets/js/02341ef2.76f0a379.js
similarity index 98%
rename from assets/js/02341ef2.5a4f1f57.js
rename to assets/js/02341ef2.76f0a379.js
index 52b52d9c..e6ceed95 100644
--- a/assets/js/02341ef2.5a4f1f57.js
+++ b/assets/js/02341ef2.76f0a379.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[6],{68:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.11","label":"1.2.11","isLast":false,"docsSidebars":{"version-1.2.11/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.11/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/1.2.11/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.11/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.11/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.11/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/1.2.11/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/choose_install":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/dev_dataset":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docusaurus":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/elastic":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/importer":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/intro":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/logstash":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/nfdump":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/tstat":"version-1.2.11/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[5],{67:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.11","label":"1.2.11","isLast":false,"docsSidebars":{"version-1.2.11/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.11/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/1.2.11/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.11/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.11/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.11/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/1.2.11/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/choose_install":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/dev_dataset":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/devel/docusaurus":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/elastic":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/importer":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/intro":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/logstash":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/nfdump":"version-1.2.11/Pipeline","/netsage-pipeline/docs/1.2.11/pipeline/tstat":"version-1.2.11/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/0271802f.b404da72.js b/assets/js/0271802f.e2ca0056.js
similarity index 98%
rename from assets/js/0271802f.b404da72.js
rename to assets/js/0271802f.e2ca0056.js
index 5728dd37..035a5c7a 100644
--- a/assets/js/0271802f.b404da72.js
+++ b/assets/js/0271802f.e2ca0056.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[7],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<o;c++)s[c]=n[c];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},69:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.10/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-1.2.10/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/logstash.md",version:"1.2.10",sidebar_label:"Logstash",sidebar:"version-1.2.10/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list, 2) the ability to change the sensor name for flows from a specified sensor which go through a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[6],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<o;c++)s[c]=n[c];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},68:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.10/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-1.2.10/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/logstash.md",version:"1.2.10",sidebar_label:"Logstash",sidebar:"version-1.2.10/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list, 2) the ability to change the sensor name for flows from a specified sensor which go through a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0327c8bf.c122f958.js b/assets/js/0327c8bf.4ed718d8.js
similarity index 99%
rename from assets/js/0327c8bf.c122f958.js
rename to assets/js/0327c8bf.4ed718d8.js
index 92a6e95d..cd0974d5 100644
--- a/assets/js/0327c8bf.c122f958.js
+++ b/assets/js/0327c8bf.4ed718d8.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[8],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"},70:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.11/deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",version:"1.2.11",sidebar_label:"Manual Installation",sidebar:"version-1.2.11/Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[7],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"},69:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(228)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.11/deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",version:"1.2.11",sidebar_label:"Manual Installation",sidebar:"version-1.2.11/Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0381afdd.426b41ab.js b/assets/js/0381afdd.de4385a8.js
similarity index 97%
rename from assets/js/0381afdd.426b41ab.js
rename to assets/js/0381afdd.de4385a8.js
index fc4575d6..3af9c852 100644
--- a/assets/js/0381afdd.426b41ab.js
+++ b/assets/js/0381afdd.de4385a8.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[9],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},71:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.7/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.7/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.7/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/documentation_guide.md",version:"1.2.7",sidebar_label:"Documentation Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[8],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},70:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.7/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.7/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.7/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/documentation_guide.md",version:"1.2.7",sidebar_label:"Documentation Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/05c6fd0e.27777f60.js b/assets/js/05c6fd0e.27777f60.js
new file mode 100644
index 00000000..a5573599
--- /dev/null
+++ b/assets/js/05c6fd0e.27777f60.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[9],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},71:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.11/deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/versioned_docs/version-1.2.11/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/choosing.md",version:"1.2.11",sidebar_label:"Choose Install",sidebar:"version-1.2.11/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/05c6fd0e.586b8e3b.js b/assets/js/05c6fd0e.586b8e3b.js
deleted file mode 100644
index 7f72f7d5..00000000
--- a/assets/js/05c6fd0e.586b8e3b.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[10],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},72:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.11/deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/versioned_docs/version-1.2.11/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/choosing.md",version:"1.2.11",sidebar_label:"Choose Install",sidebar:"version-1.2.11/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/066f8364.0e023e16.js b/assets/js/066f8364.bfb615a4.js
similarity index 96%
rename from assets/js/066f8364.0e023e16.js
rename to assets/js/066f8364.bfb615a4.js
index cb98c20f..0a2d2cf6 100644
--- a/assets/js/066f8364.0e023e16.js
+++ b/assets/js/066f8364.bfb615a4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[11],{232:function(e,t,r){"use strict";r.d(t,"a",(function(){return u})),r.d(t,"b",(function(){return m}));var n=r(0),i=r.n(n);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){o(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function p(e,t){if(null==e)return{};var r,n,i=function(e,t){if(null==e)return{};var r,n,i={},o=Object.keys(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,a=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(r),f=n,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return r?i.a.createElement(m,l(l({ref:t},c),{},{components:r})):i.a.createElement(m,l({ref:t},c))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,a=new Array(o);a[0]=f;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<o;c++)a[c]=r[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,r)}f.displayName="MDXCreateElement"},73:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return l})),r.d(t,"toc",(function(){return p})),r.d(t,"default",(function(){return s}));var n=r(3),i=r(7),o=(r(0),r(232)),a={id:"pipeline_importer",title:"Importer",sidebar_label:"Importer"},l={unversionedId:"pipeline_importer",id:"version-1.2.5/pipeline_importer",isDocsHomePage:!1,title:"Importer",description:"Importer",source:"@site/versioned_docs/version-1.2.5/pipeline_importer.md",slug:"/pipeline_importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_importer.md",version:"1.2.5",sidebar_label:"Importer",sidebar:"version-1.2.5/Pipeline",previous:{title:"Pipeline",permalink:"/netsage-pipeline/docs/1.2.5/pipeline"},next:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash"}},p=[{value:"Importer",id:"importer",children:[]},{value:"Importer",id:"importer-1",children:[{value:"Configuration",id:"configuration",children:[]},{value:"Internals",id:"internals",children:[]}]}],c={toc:p};function s(e){var t=e.components,r=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(n.a)({},c,r,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."),Object(o.b)("h2",{id:"importer-1"},"Importer"),Object(o.b)("p",null,"NOTE: Importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"configuration files for the importer are ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_netflow_importer.xml")," and ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," in ",Object(o.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifer/"),". Comments in the files briefly describe the options."),Object(o.b)("p",null,"Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. "),Object(o.b)("h3",{id:"internals"},"Internals"),Object(o.b)("p",null,"The importer uses the nfdump command with -a to aggregate within the file, and -L ",Object(o.b)("inlineCode",{parentName:"p"},"threshold")," to throw out any flows under a flow size threshold. "),Object(o.b)("p",null,"For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.\nA ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages."),Object(o.b)("p",null,"All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows."))}s.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[10],{228:function(e,t,r){"use strict";r.d(t,"a",(function(){return u})),r.d(t,"b",(function(){return m}));var n=r(0),i=r.n(n);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){o(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function p(e,t){if(null==e)return{};var r,n,i=function(e,t){if(null==e)return{};var r,n,i={},o=Object.keys(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,a=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(r),f=n,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return r?i.a.createElement(m,l(l({ref:t},c),{},{components:r})):i.a.createElement(m,l({ref:t},c))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,a=new Array(o);a[0]=f;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<o;c++)a[c]=r[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,r)}f.displayName="MDXCreateElement"},72:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return l})),r.d(t,"toc",(function(){return p})),r.d(t,"default",(function(){return s}));var n=r(3),i=r(7),o=(r(0),r(228)),a={id:"pipeline_importer",title:"Importer",sidebar_label:"Importer"},l={unversionedId:"pipeline_importer",id:"version-1.2.5/pipeline_importer",isDocsHomePage:!1,title:"Importer",description:"Importer",source:"@site/versioned_docs/version-1.2.5/pipeline_importer.md",slug:"/pipeline_importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_importer.md",version:"1.2.5",sidebar_label:"Importer",sidebar:"version-1.2.5/Pipeline",previous:{title:"Pipeline",permalink:"/netsage-pipeline/docs/1.2.5/pipeline"},next:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash"}},p=[{value:"Importer",id:"importer",children:[]},{value:"Importer",id:"importer-1",children:[{value:"Configuration",id:"configuration",children:[]},{value:"Internals",id:"internals",children:[]}]}],c={toc:p};function s(e){var t=e.components,r=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(n.a)({},c,r,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."),Object(o.b)("h2",{id:"importer-1"},"Importer"),Object(o.b)("p",null,"NOTE: Importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"configuration files for the importer are ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_netflow_importer.xml")," and ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," in ",Object(o.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifer/"),". Comments in the files briefly describe the options."),Object(o.b)("p",null,"Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. "),Object(o.b)("h3",{id:"internals"},"Internals"),Object(o.b)("p",null,"The importer uses the nfdump command with -a to aggregate within the file, and -L ",Object(o.b)("inlineCode",{parentName:"p"},"threshold")," to throw out any flows under a flow size threshold. "),Object(o.b)("p",null,"For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.\nA ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages."),Object(o.b)("p",null,"All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows."))}s.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/07faddd5.24f5303f.js b/assets/js/07faddd5.b2854b31.js
similarity index 97%
rename from assets/js/07faddd5.24f5303f.js
rename to assets/js/07faddd5.b2854b31.js
index 88b53fa6..b0561bb9 100644
--- a/assets/js/07faddd5.24f5303f.js
+++ b/assets/js/07faddd5.b2854b31.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[12],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},74:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(232)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.7/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.7/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/intro.md",version:"1.2.7",sidebar_label:"Intro",sidebar:"version-1.2.7/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers can be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"[Importer]",":  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   ",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/pipeline/importer"},"Doc"),". ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"In github"),"."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash"),": Performs a variety of operations on the flow data to transform it and add additional information.  ",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},"Doc"),"."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow or netflow sensor, or one of each (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[11],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},73:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(228)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.7/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.7/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/intro.md",version:"1.2.7",sidebar_label:"Intro",sidebar:"version-1.2.7/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers can be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"[Importer]",":  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   ",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/pipeline/importer"},"Doc"),". ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"In github"),"."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash"),": Performs a variety of operations on the flow data to transform it and add additional information.  ",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},"Doc"),"."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow or netflow sensor, or one of each (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/080a21b4.34495c8a.js b/assets/js/080a21b4.616746c2.js
similarity index 96%
rename from assets/js/080a21b4.34495c8a.js
rename to assets/js/080a21b4.616746c2.js
index b73a7d8b..d51d448f 100644
--- a/assets/js/080a21b4.34495c8a.js
+++ b/assets/js/080a21b4.616746c2.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[13],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"},75:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/docs/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/next/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_first_steps.md",version:"current"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[12],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"},74:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/docs/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/next/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_first_steps.md",version:"current"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/092c2d1d.97793caf.js b/assets/js/092c2d1d.97793caf.js
new file mode 100644
index 00000000..74714359
--- /dev/null
+++ b/assets/js/092c2d1d.97793caf.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[13],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},75:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-2.1.0/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-2.1.0/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/nfdump.md",version:"2.1.0",sidebar_label:"Sflow/Netflow Data",sidebar:"version-2.1.0/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0b8e461a.4f5dc652.js b/assets/js/0b8e461a.4a1a6155.js
similarity index 97%
rename from assets/js/0b8e461a.4f5dc652.js
rename to assets/js/0b8e461a.4a1a6155.js
index 98d367e5..b00120e7 100644
--- a/assets/js/0b8e461a.4f5dc652.js
+++ b/assets/js/0b8e461a.4a1a6155.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[14],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},76:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-1.2.10/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-1.2.10/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/tstat.md",version:"1.2.10",sidebar_label:"Tstat Data",sidebar:"version-1.2.10/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[14],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},76:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-1.2.10/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-1.2.10/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/tstat.md",version:"1.2.10",sidebar_label:"Tstat Data",sidebar:"version-1.2.10/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0d6d53ac.2ae3b199.js b/assets/js/0d6d53ac.2ae3b199.js
deleted file mode 100644
index d1d19bef..00000000
--- a/assets/js/0d6d53ac.2ae3b199.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[15],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"},77:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(232)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-2.0.0/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-2.0.0/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/documentation_guide.md",version:"2.0.0",sidebar_label:"Docusaurus",sidebar:"version-2.0.0/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json"),Object(i.b)("li",{parentName:"ul"},"change 1.2.6 in docusaurus.config.js back to 1.2.5")),Object(i.b)("p",null,"Then git add, commit, and push"),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/0dd3dba6.3dbbc1f3.js b/assets/js/0dd3dba6.3dbbc1f3.js
new file mode 100644
index 00000000..f667f8f0
--- /dev/null
+++ b/assets/js/0dd3dba6.3dbbc1f3.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[15],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(a,".").concat(b)]||p[b]||u[b]||i;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,a[1]=c;for(var l=2;l<i;l++)a[l]=n[l];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},77:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-2.1.0/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-2.1.0/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/docker.md",version:"2.1.0",sidebar_label:"Docker Dev Guide",sidebar:"version-2.1.0/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/devel/docusaurus"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],l={toc:s};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/1.bb3bc4d5.js b/assets/js/1.03db48d6.js
similarity index 84%
rename from assets/js/1.bb3bc4d5.js
rename to assets/js/1.03db48d6.js
index 2e527e01..980c4dcf 100644
--- a/assets/js/1.bb3bc4d5.js
+++ b/assets/js/1.03db48d6.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[1],{233:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(244);Object.defineProperty(t,"useThemeConfig",{enumerable:!0,get:function(){return r.useThemeConfig}});var o=n(258);Object.defineProperty(t,"useAlternatePageUtils",{enumerable:!0,get:function(){return o.useAlternatePageUtils}});var u=n(259);Object.defineProperty(t,"docVersionSearchTag",{enumerable:!0,get:function(){return u.docVersionSearchTag}}),Object.defineProperty(t,"DEFAULT_SEARCH_TAG",{enumerable:!0,get:function(){return u.DEFAULT_SEARCH_TAG}});var i=n(245);Object.defineProperty(t,"isDocsPluginEnabled",{enumerable:!0,get:function(){return i.isDocsPluginEnabled}});var a=n(263);Object.defineProperty(t,"isSamePath",{enumerable:!0,get:function(){return a.isSamePath}});var s=n(264);Object.defineProperty(t,"useTitleFormatter",{enumerable:!0,get:function(){return s.useTitleFormatter}});var c=n(265);Object.defineProperty(t,"usePluralForm",{enumerable:!0,get:function(){return c.usePluralForm}});var l=n(266);Object.defineProperty(t,"useDocsPreferredVersion",{enumerable:!0,get:function(){return l.useDocsPreferredVersion}}),Object.defineProperty(t,"useDocsPreferredVersionByPluginId",{enumerable:!0,get:function(){return l.useDocsPreferredVersionByPluginId}});var f=n(246);Object.defineProperty(t,"DocsPreferredVersionContextProvider",{enumerable:!0,get:function(){return f.DocsPreferredVersionContextProvider}})},234:function(e,t,n){"use strict";function r(e){var t,n,o="";if("string"==typeof e||"number"==typeof e)o+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;t<e.length;t++)e[t]&&(n=r(e[t]))&&(o&&(o+=" "),o+=n);else for(t in e)e[t]&&(o&&(o+=" "),o+=t);return o}t.a=function(){for(var e,t,n=0,o="";n<arguments.length;)(e=arguments[n++])&&(t=r(e))&&(o&&(o+=" "),o+=t);return o}},235:function(e,t,n){"use strict";n.d(t,"b",(function(){return f})),n.d(t,"a",(function(){return d}));var r=n(0),o=n.n(r),u=/{\w+}/g,i="{}";function a(e,t){var n=[],r=e.replace(u,(function(e){var r=e.substr(1,e.length-2),u=null==t?void 0:t[r];if(void 0!==u){var a=o.a.isValidElement(u)?u:String(u);return n.push(a),i}return e}));return 0===n.length?e:n.every((function(e){return"string"==typeof e}))?r.split(i).reduce((function(e,t,r){var o;return e.concat(t).concat(null!==(o=n[r])&&void 0!==o?o:"")}),""):r.split(i).reduce((function(e,t,r){return[].concat(e,[o.a.createElement(o.a.Fragment,{key:r},t,n[r])])}),[])}function s(e){return a(e.children,e.values)}var c=n(26);function l(e){var t,n=e.id,r=e.message;return null!==(t=c[null!=n?n:r])&&void 0!==t?t:r}function f(e,t){var n,r=e.message;return a(null!==(n=l({message:r,id:e.id}))&&void 0!==n?n:r,t)}function d(e){var t,n=e.children,r=e.id,u=e.values,i=null!==(t=l({message:n,id:r}))&&void 0!==t?t:n;return o.a.createElement(s,{values:u},i)}},236:function(e,t,n){try{e.exports=n(260)}catch(o){var r={};e.exports={useAllDocsData:function(){return r},useActivePluginAndVersion:function(){}}}},237:function(e,t,n){"use strict";var r=n(0),o=n.n(r),u=n(10),i=n(242),a=n(8),s=Object(r.createContext)({collectLink:function(){}}),c=n(238),l=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&t.indexOf(r)<0&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols){var o=0;for(r=Object.getOwnPropertySymbols(e);o<r.length;o++)t.indexOf(r[o])<0&&Object.prototype.propertyIsEnumerable.call(e,r[o])&&(n[r[o]]=e[r[o]])}return n};t.a=function(e){var t,n,f,d=e.isNavLink,v=e.to,g=e.href,m=e.activeClassName,p=e.isActive,h=e["data-noBrokenLinkCheck"],b=e.autoAddBaseUrl,P=void 0===b||b,D=l(e,["isNavLink","to","href","activeClassName","isActive","data-noBrokenLinkCheck","autoAddBaseUrl"]),_=Object(c.b)().withBaseUrl,y=Object(r.useContext)(s),O=v||g,V=Object(i.a)(O),A=null==O?void 0:O.replace("pathname://",""),j=void 0!==A?(n=A,P&&function(e){return e.startsWith("/")}(n)?_(n):n):void 0,w=Object(r.useRef)(!1),E=d?u.e:u.c,C=a.a.canUseIntersectionObserver;Object(r.useEffect)((function(){return!C&&V&&window.docusaurus.prefetch(j),function(){C&&f&&f.disconnect()}}),[j,C,V]);var x=null!==(t=null==j?void 0:j.startsWith("#"))&&void 0!==t&&t,L=!j||!V||x;return j&&V&&!x&&!h&&y.collectLink(j),L?o.a.createElement("a",Object.assign({href:j},O&&!V&&{target:"_blank",rel:"noopener noreferrer"},D)):o.a.createElement(E,Object.assign({},D,{onMouseEnter:function(){w.current||(window.docusaurus.preload(j),w.current=!0)},innerRef:function(e){var t,n;C&&e&&V&&(t=e,n=function(){window.docusaurus.prefetch(j)},(f=new window.IntersectionObserver((function(e){e.forEach((function(e){t===e.target&&(e.isIntersecting||e.intersectionRatio>0)&&(f.unobserve(t),f.disconnect(),n())}))}))).observe(t))},to:j||""},d&&{isActive:p,activeClassName:m}))}},238:function(e,t,n){"use strict";n.d(t,"b",(function(){return u})),n.d(t,"a",(function(){return i}));var r=n(16),o=n(242);function u(){var e=Object(r.default)().siteConfig,t=(e=void 0===e?{}:e).baseUrl,n=void 0===t?"/":t,u=e.url;return{withBaseUrl:function(e,t){return function(e,t,n,r){var u=void 0===r?{}:r,i=u.forcePrependBaseUrl,a=void 0!==i&&i,s=u.absolute,c=void 0!==s&&s;if(!n)return n;if(n.startsWith("#"))return n;if(Object(o.b)(n))return n;if(a)return t+n;var l=n.startsWith(t)?n:t+n.replace(/^\//,"");return c?e+l:l}(u,n,e,t)}}}function i(e,t){return void 0===t&&(t={}),(0,u().withBaseUrl)(e,t)}},242:function(e,t,n){"use strict";function r(e){return!0===/^(\w*:|\/\/)/.test(e)}function o(e){return void 0!==e&&!r(e)}n.d(t,"b",(function(){return r})),n.d(t,"a",(function(){return o}))},244:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useThemeConfig=void 0;var o=r(n(16));t.useThemeConfig=function(){return o.default().siteConfig.themeConfig}},245:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.isDocsPluginEnabled=void 0;var r=n(236);t.isDocsPluginEnabled=!!r.useAllDocsData},246:function(e,t,n){"use strict";var r=this&&this.__createBinding||(Object.create?function(e,t,n,r){void 0===r&&(r=n),Object.defineProperty(e,r,{enumerable:!0,get:function(){return t[n]}})}:function(e,t,n,r){void 0===r&&(r=n),e[r]=t[n]}),o=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),u=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)"default"!==n&&Object.hasOwnProperty.call(e,n)&&r(t,e,n);return o(t,e),t},i=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useDocsPreferredVersionContext=t.DocsPreferredVersionContextProvider=void 0;var a=u(n(0)),s=n(244),c=n(245),l=n(236),f=i(n(267));function d(e){var t=e.pluginIds,n=e.versionPersistence,r=e.allDocsData;var o={};return t.forEach((function(e){o[e]=function(e){var t=f.default.read(e,n);return r[e].versions.some((function(e){return e.name===t}))?{preferredVersionName:t}:(f.default.clear(e,n),{preferredVersionName:null})}(e)})),o}function v(){var e=l.useAllDocsData(),t=s.useThemeConfig().docs.versionPersistence,n=a.useMemo((function(){return Object.keys(e)}),[e]),r=a.useState((function(){return function(e){var t={};return e.forEach((function(e){t[e]={preferredVersionName:null}})),t}(n)})),o=r[0],u=r[1];return a.useEffect((function(){u(d({allDocsData:e,versionPersistence:t,pluginIds:n}))}),[e,t,n]),[o,a.useMemo((function(){return{savePreferredVersion:function(e,n){f.default.save(e,t,n),u((function(t){var r;return Object.assign(Object.assign({},t),((r={})[e]={preferredVersionName:n},r))}))}}}),[u])]}var g=a.createContext(null);function m(e){var t=e.children,n=v();return a.default.createElement(g.Provider,{value:n},t)}t.DocsPreferredVersionContextProvider=function(e){var t=e.children;return c.isDocsPluginEnabled?a.default.createElement(m,null,t):a.default.createElement(a.default.Fragment,null,t)},t.useDocsPreferredVersionContext=function(){var e=a.useContext(g);if(!e)throw new Error("Can't find docs preferred context, maybe you forgot to use the DocsPreferredVersionContextProvider ?");return e}},257:function(e,t,n){"use strict";n.d(t,"a",(function(){return s}));var r=n(0),o=n.n(r),u=n(25),i=n(238),a=n(233);function s(e){var t=e.title,n=e.description,r=e.keywords,s=e.image,c=Object(a.useTitleFormatter)(t),l=Object(i.a)(s,{absolute:!0});return o.a.createElement(u.a,null,t&&o.a.createElement("title",null,c),t&&o.a.createElement("meta",{property:"og:title",content:c}),n&&o.a.createElement("meta",{name:"description",content:n}),n&&o.a.createElement("meta",{property:"og:description",content:n}),r&&o.a.createElement("meta",{name:"keywords",content:Array.isArray(r)?r.join(","):r}),s&&o.a.createElement("meta",{property:"og:image",content:l}),s&&o.a.createElement("meta",{name:"twitter:image",content:l}),s&&o.a.createElement("meta",{name:"twitter:card",content:"summary_large_image"}))}},258:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useAlternatePageUtils=void 0;var o=r(n(16)),u=n(23);t.useAlternatePageUtils=function(){var e=o.default(),t=e.siteConfig,n=t.baseUrl,r=t.url,i=e.i18n,a=i.defaultLocale,s=i.currentLocale,c=u.useLocation().pathname,l=s===a?n:n.replace("/"+s+"/","/"),f=c.replace(n,"");return{createUrl:function(e){var t=e.locale;return""+(e.fullyQualified?r:"")+function(e){return e===a?""+l:""+l+e+"/"}(t)+f}}}},259:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.docVersionSearchTag=t.DEFAULT_SEARCH_TAG=void 0,t.DEFAULT_SEARCH_TAG="default",t.docVersionSearchTag=function(e,t){return"docs-"+e+"-"+t}},260:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.useDocVersionSuggestions=t.useActiveDocContext=t.useActiveVersion=t.useLatestVersion=t.useVersions=t.useActivePluginAndVersion=t.useActivePlugin=t.useDocsData=t.useAllDocsData=void 0;var r=n(23),o=n(261),u=n(262);t.useAllDocsData=function(){return o.useAllPluginInstancesData("docusaurus-plugin-content-docs")},t.useDocsData=function(e){return o.usePluginData("docusaurus-plugin-content-docs",e)},t.useActivePlugin=function(e){void 0===e&&(e={});var n=t.useAllDocsData(),o=r.useLocation().pathname;return u.getActivePlugin(n,o,e)},t.useActivePluginAndVersion=function(e){void 0===e&&(e={});var n=t.useActivePlugin(e),o=r.useLocation().pathname;if(n)return{activePlugin:n,activeVersion:u.getActiveVersion(n.pluginData,o)}},t.useVersions=function(e){return t.useDocsData(e).versions},t.useLatestVersion=function(e){var n=t.useDocsData(e);return u.getLatestVersion(n)},t.useActiveVersion=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getActiveVersion(n,o)},t.useActiveDocContext=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getActiveDocContext(n,o)},t.useDocVersionSuggestions=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getDocVersionSuggestions(n,o)}},261:function(e,t,n){"use strict";n.r(t),n.d(t,"default",(function(){return o})),n.d(t,"useAllPluginInstancesData",(function(){return u})),n.d(t,"usePluginData",(function(){return i}));var r=n(16);function o(){var e=Object(r.default)().globalData;if(!e)throw new Error("Docusaurus global data not found");return e}function u(e){var t=o()[e];if(!t)throw new Error("Docusaurus plugin global data not found for pluginName="+e);return t}function i(e,t){void 0===t&&(t="default");var n=u(e)[t];if(!n)throw new Error("Docusaurus plugin global data not found for pluginName="+e+" and pluginId="+t);return n}},262:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getDocVersionSuggestions=t.getActiveDocContext=t.getActiveVersion=t.getLatestVersion=t.getActivePlugin=void 0;var r=n(23);t.getActivePlugin=function(e,t,n){void 0===n&&(n={});var o=Object.entries(e).find((function(e){e[0];var n=e[1];return!!r.matchPath(t,{path:n.path,exact:!1,strict:!1})})),u=o?{pluginId:o[0],pluginData:o[1]}:void 0;if(!u&&n.failfast)throw new Error("Can't find active docs plugin for pathname="+t+", while it was expected to be found. Maybe you tried to use a docs feature that can only be used on a docs-related page? Existing docs plugin paths are: "+Object.values(e).map((function(e){return e.path})).join(", "));return u},t.getLatestVersion=function(e){return e.versions.find((function(e){return e.isLast}))},t.getActiveVersion=function(e,n){var o=t.getLatestVersion(e);return[].concat(e.versions.filter((function(e){return e!==o})),[o]).find((function(e){return!!r.matchPath(n,{path:e.path,exact:!1,strict:!1})}))},t.getActiveDocContext=function(e,n){var o,u,i=t.getActiveVersion(e,n),a=null==i?void 0:i.docs.find((function(e){return!!r.matchPath(n,{path:e.path,exact:!0,strict:!1})}));return{activeVersion:i,activeDoc:a,alternateDocVersions:a?(o=a.id,u={},e.versions.forEach((function(e){e.docs.forEach((function(t){t.id===o&&(u[e.name]=t)}))})),u):{}}},t.getDocVersionSuggestions=function(e,n){var r=t.getLatestVersion(e),o=t.getActiveDocContext(e,n),u=o.activeVersion!==r;return{latestDocSuggestion:u?null==o?void 0:o.alternateDocVersions[r.name]:void 0,latestVersionSuggestion:u?r:void 0}}},263:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.isSamePath=void 0,t.isSamePath=function(e,t){var n=function(e){return!e||(null==e?void 0:e.endsWith("/"))?e:e+"/"};return n(e)===n(t)}},264:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useTitleFormatter=void 0;var o=r(n(16));t.useTitleFormatter=function(e){var t=o.default().siteConfig,n=void 0===t?{}:t,r=n.title,u=n.titleDelimiter,i=void 0===u?"|":u;return e&&e.trim().length?e.trim()+" "+i+" "+r:r}},265:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.usePluralForm=void 0;var o=n(0),u=r(n(16)),i=["zero","one","two","few","many","other"];function a(e){return i.filter((function(t){return e.includes(t)}))}var s={locale:"en",pluralForms:a(["one","other"]),select:function(e){return 1===e?"one":"other"}};function c(){var e=u.default().i18n.currentLocale;return o.useMemo((function(){if(!Intl.PluralRules)return console.error("Intl.PluralRules not available!\nDocusaurus will fallback to a default/fallback (English) Intl.PluralRules implementation.\n        "),s;try{return t=e,n=new Intl.PluralRules(t),{locale:t,pluralForms:a(n.resolvedOptions().pluralCategories),select:function(e){return n.select(e)}}}catch(r){return console.error("Failed to use Intl.PluralRules for locale="+e+".\nDocusaurus will fallback to a default/fallback (English) Intl.PluralRules implementation.\n"),s}var t,n}),[e])}t.usePluralForm=function(){var e=c();return{selectMessage:function(t,n){return function(e,t,n){var r=e.split("|");if(1===r.length)return r[0];r.length>n.pluralForms.length&&console.error("For locale="+n.locale+", a maximum of "+n.pluralForms.length+" plural forms are expected ("+n.pluralForms+"), but the message contains "+r.length+" plural forms: "+e+" ");var o=n.select(t),u=n.pluralForms.indexOf(o);return r[Math.min(u,r.length-1)]}(n,t,e)}}}},266:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.useDocsPreferredVersionByPluginId=t.useDocsPreferredVersion=void 0;var r=n(0),o=n(246),u=n(236),i=n(268);t.useDocsPreferredVersion=function(e){void 0===e&&(e=i.DEFAULT_PLUGIN_ID);var t=u.useDocsData(e),n=o.useDocsPreferredVersionContext(),a=n[0],s=n[1],c=a[e].preferredVersionName;return{preferredVersion:c?t.versions.find((function(e){return e.name===c})):null,savePreferredVersionName:r.useCallback((function(t){s.savePreferredVersion(e,t)}),[s])}},t.useDocsPreferredVersionByPluginId=function(){var e=u.useAllDocsData(),t=o.useDocsPreferredVersionContext()[0],n=Object.keys(e),r={};return n.forEach((function(n){r[n]=function(n){var r=e[n],o=t[n].preferredVersionName;return o?r.versions.find((function(e){return e.name===o})):null}(n)})),r}},267:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=function(e){return"docs-preferred-version-"+e},o={save:function(e,t,n){"none"===t||window.localStorage.setItem(r(e),n)},read:function(e,t){return"none"===t?null:window.localStorage.getItem(r(e))},clear:function(e,t){"none"===t||window.localStorage.removeItem(r(e))}};t.default=o},268:function(e,t,n){"use strict";n.r(t),n.d(t,"DEFAULT_PLUGIN_ID",(function(){return r}));var r="default"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[1],{229:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(240);Object.defineProperty(t,"useThemeConfig",{enumerable:!0,get:function(){return r.useThemeConfig}});var o=n(254);Object.defineProperty(t,"useAlternatePageUtils",{enumerable:!0,get:function(){return o.useAlternatePageUtils}});var u=n(255);Object.defineProperty(t,"docVersionSearchTag",{enumerable:!0,get:function(){return u.docVersionSearchTag}}),Object.defineProperty(t,"DEFAULT_SEARCH_TAG",{enumerable:!0,get:function(){return u.DEFAULT_SEARCH_TAG}});var i=n(241);Object.defineProperty(t,"isDocsPluginEnabled",{enumerable:!0,get:function(){return i.isDocsPluginEnabled}});var a=n(259);Object.defineProperty(t,"isSamePath",{enumerable:!0,get:function(){return a.isSamePath}});var s=n(260);Object.defineProperty(t,"useTitleFormatter",{enumerable:!0,get:function(){return s.useTitleFormatter}});var c=n(261);Object.defineProperty(t,"usePluralForm",{enumerable:!0,get:function(){return c.usePluralForm}});var l=n(262);Object.defineProperty(t,"useDocsPreferredVersion",{enumerable:!0,get:function(){return l.useDocsPreferredVersion}}),Object.defineProperty(t,"useDocsPreferredVersionByPluginId",{enumerable:!0,get:function(){return l.useDocsPreferredVersionByPluginId}});var f=n(242);Object.defineProperty(t,"DocsPreferredVersionContextProvider",{enumerable:!0,get:function(){return f.DocsPreferredVersionContextProvider}})},230:function(e,t,n){"use strict";function r(e){var t,n,o="";if("string"==typeof e||"number"==typeof e)o+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;t<e.length;t++)e[t]&&(n=r(e[t]))&&(o&&(o+=" "),o+=n);else for(t in e)e[t]&&(o&&(o+=" "),o+=t);return o}t.a=function(){for(var e,t,n=0,o="";n<arguments.length;)(e=arguments[n++])&&(t=r(e))&&(o&&(o+=" "),o+=t);return o}},231:function(e,t,n){"use strict";n.d(t,"b",(function(){return f})),n.d(t,"a",(function(){return d}));var r=n(0),o=n.n(r),u=/{\w+}/g,i="{}";function a(e,t){var n=[],r=e.replace(u,(function(e){var r=e.substr(1,e.length-2),u=null==t?void 0:t[r];if(void 0!==u){var a=o.a.isValidElement(u)?u:String(u);return n.push(a),i}return e}));return 0===n.length?e:n.every((function(e){return"string"==typeof e}))?r.split(i).reduce((function(e,t,r){var o;return e.concat(t).concat(null!==(o=n[r])&&void 0!==o?o:"")}),""):r.split(i).reduce((function(e,t,r){return[].concat(e,[o.a.createElement(o.a.Fragment,{key:r},t,n[r])])}),[])}function s(e){return a(e.children,e.values)}var c=n(26);function l(e){var t,n=e.id,r=e.message;return null!==(t=c[null!=n?n:r])&&void 0!==t?t:r}function f(e,t){var n,r=e.message;return a(null!==(n=l({message:r,id:e.id}))&&void 0!==n?n:r,t)}function d(e){var t,n=e.children,r=e.id,u=e.values,i=null!==(t=l({message:n,id:r}))&&void 0!==t?t:n;return o.a.createElement(s,{values:u},i)}},232:function(e,t,n){try{e.exports=n(256)}catch(o){var r={};e.exports={useAllDocsData:function(){return r},useActivePluginAndVersion:function(){}}}},233:function(e,t,n){"use strict";var r=n(0),o=n.n(r),u=n(10),i=n(238),a=n(8),s=Object(r.createContext)({collectLink:function(){}}),c=n(234),l=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&t.indexOf(r)<0&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols){var o=0;for(r=Object.getOwnPropertySymbols(e);o<r.length;o++)t.indexOf(r[o])<0&&Object.prototype.propertyIsEnumerable.call(e,r[o])&&(n[r[o]]=e[r[o]])}return n};t.a=function(e){var t,n,f,d=e.isNavLink,v=e.to,g=e.href,m=e.activeClassName,p=e.isActive,h=e["data-noBrokenLinkCheck"],b=e.autoAddBaseUrl,P=void 0===b||b,D=l(e,["isNavLink","to","href","activeClassName","isActive","data-noBrokenLinkCheck","autoAddBaseUrl"]),_=Object(c.b)().withBaseUrl,y=Object(r.useContext)(s),O=v||g,V=Object(i.a)(O),A=null==O?void 0:O.replace("pathname://",""),j=void 0!==A?(n=A,P&&function(e){return e.startsWith("/")}(n)?_(n):n):void 0,w=Object(r.useRef)(!1),E=d?u.e:u.c,C=a.a.canUseIntersectionObserver;Object(r.useEffect)((function(){return!C&&V&&window.docusaurus.prefetch(j),function(){C&&f&&f.disconnect()}}),[j,C,V]);var x=null!==(t=null==j?void 0:j.startsWith("#"))&&void 0!==t&&t,L=!j||!V||x;return j&&V&&!x&&!h&&y.collectLink(j),L?o.a.createElement("a",Object.assign({href:j},O&&!V&&{target:"_blank",rel:"noopener noreferrer"},D)):o.a.createElement(E,Object.assign({},D,{onMouseEnter:function(){w.current||(window.docusaurus.preload(j),w.current=!0)},innerRef:function(e){var t,n;C&&e&&V&&(t=e,n=function(){window.docusaurus.prefetch(j)},(f=new window.IntersectionObserver((function(e){e.forEach((function(e){t===e.target&&(e.isIntersecting||e.intersectionRatio>0)&&(f.unobserve(t),f.disconnect(),n())}))}))).observe(t))},to:j||""},d&&{isActive:p,activeClassName:m}))}},234:function(e,t,n){"use strict";n.d(t,"b",(function(){return u})),n.d(t,"a",(function(){return i}));var r=n(16),o=n(238);function u(){var e=Object(r.default)().siteConfig,t=(e=void 0===e?{}:e).baseUrl,n=void 0===t?"/":t,u=e.url;return{withBaseUrl:function(e,t){return function(e,t,n,r){var u=void 0===r?{}:r,i=u.forcePrependBaseUrl,a=void 0!==i&&i,s=u.absolute,c=void 0!==s&&s;if(!n)return n;if(n.startsWith("#"))return n;if(Object(o.b)(n))return n;if(a)return t+n;var l=n.startsWith(t)?n:t+n.replace(/^\//,"");return c?e+l:l}(u,n,e,t)}}}function i(e,t){return void 0===t&&(t={}),(0,u().withBaseUrl)(e,t)}},238:function(e,t,n){"use strict";function r(e){return!0===/^(\w*:|\/\/)/.test(e)}function o(e){return void 0!==e&&!r(e)}n.d(t,"b",(function(){return r})),n.d(t,"a",(function(){return o}))},240:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useThemeConfig=void 0;var o=r(n(16));t.useThemeConfig=function(){return o.default().siteConfig.themeConfig}},241:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.isDocsPluginEnabled=void 0;var r=n(232);t.isDocsPluginEnabled=!!r.useAllDocsData},242:function(e,t,n){"use strict";var r=this&&this.__createBinding||(Object.create?function(e,t,n,r){void 0===r&&(r=n),Object.defineProperty(e,r,{enumerable:!0,get:function(){return t[n]}})}:function(e,t,n,r){void 0===r&&(r=n),e[r]=t[n]}),o=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),u=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)"default"!==n&&Object.hasOwnProperty.call(e,n)&&r(t,e,n);return o(t,e),t},i=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useDocsPreferredVersionContext=t.DocsPreferredVersionContextProvider=void 0;var a=u(n(0)),s=n(240),c=n(241),l=n(232),f=i(n(263));function d(e){var t=e.pluginIds,n=e.versionPersistence,r=e.allDocsData;var o={};return t.forEach((function(e){o[e]=function(e){var t=f.default.read(e,n);return r[e].versions.some((function(e){return e.name===t}))?{preferredVersionName:t}:(f.default.clear(e,n),{preferredVersionName:null})}(e)})),o}function v(){var e=l.useAllDocsData(),t=s.useThemeConfig().docs.versionPersistence,n=a.useMemo((function(){return Object.keys(e)}),[e]),r=a.useState((function(){return function(e){var t={};return e.forEach((function(e){t[e]={preferredVersionName:null}})),t}(n)})),o=r[0],u=r[1];return a.useEffect((function(){u(d({allDocsData:e,versionPersistence:t,pluginIds:n}))}),[e,t,n]),[o,a.useMemo((function(){return{savePreferredVersion:function(e,n){f.default.save(e,t,n),u((function(t){var r;return Object.assign(Object.assign({},t),((r={})[e]={preferredVersionName:n},r))}))}}}),[u])]}var g=a.createContext(null);function m(e){var t=e.children,n=v();return a.default.createElement(g.Provider,{value:n},t)}t.DocsPreferredVersionContextProvider=function(e){var t=e.children;return c.isDocsPluginEnabled?a.default.createElement(m,null,t):a.default.createElement(a.default.Fragment,null,t)},t.useDocsPreferredVersionContext=function(){var e=a.useContext(g);if(!e)throw new Error("Can't find docs preferred context, maybe you forgot to use the DocsPreferredVersionContextProvider ?");return e}},253:function(e,t,n){"use strict";n.d(t,"a",(function(){return s}));var r=n(0),o=n.n(r),u=n(25),i=n(234),a=n(229);function s(e){var t=e.title,n=e.description,r=e.keywords,s=e.image,c=Object(a.useTitleFormatter)(t),l=Object(i.a)(s,{absolute:!0});return o.a.createElement(u.a,null,t&&o.a.createElement("title",null,c),t&&o.a.createElement("meta",{property:"og:title",content:c}),n&&o.a.createElement("meta",{name:"description",content:n}),n&&o.a.createElement("meta",{property:"og:description",content:n}),r&&o.a.createElement("meta",{name:"keywords",content:Array.isArray(r)?r.join(","):r}),s&&o.a.createElement("meta",{property:"og:image",content:l}),s&&o.a.createElement("meta",{name:"twitter:image",content:l}),s&&o.a.createElement("meta",{name:"twitter:card",content:"summary_large_image"}))}},254:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useAlternatePageUtils=void 0;var o=r(n(16)),u=n(23);t.useAlternatePageUtils=function(){var e=o.default(),t=e.siteConfig,n=t.baseUrl,r=t.url,i=e.i18n,a=i.defaultLocale,s=i.currentLocale,c=u.useLocation().pathname,l=s===a?n:n.replace("/"+s+"/","/"),f=c.replace(n,"");return{createUrl:function(e){var t=e.locale;return""+(e.fullyQualified?r:"")+function(e){return e===a?""+l:""+l+e+"/"}(t)+f}}}},255:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.docVersionSearchTag=t.DEFAULT_SEARCH_TAG=void 0,t.DEFAULT_SEARCH_TAG="default",t.docVersionSearchTag=function(e,t){return"docs-"+e+"-"+t}},256:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.useDocVersionSuggestions=t.useActiveDocContext=t.useActiveVersion=t.useLatestVersion=t.useVersions=t.useActivePluginAndVersion=t.useActivePlugin=t.useDocsData=t.useAllDocsData=void 0;var r=n(23),o=n(257),u=n(258);t.useAllDocsData=function(){return o.useAllPluginInstancesData("docusaurus-plugin-content-docs")},t.useDocsData=function(e){return o.usePluginData("docusaurus-plugin-content-docs",e)},t.useActivePlugin=function(e){void 0===e&&(e={});var n=t.useAllDocsData(),o=r.useLocation().pathname;return u.getActivePlugin(n,o,e)},t.useActivePluginAndVersion=function(e){void 0===e&&(e={});var n=t.useActivePlugin(e),o=r.useLocation().pathname;if(n)return{activePlugin:n,activeVersion:u.getActiveVersion(n.pluginData,o)}},t.useVersions=function(e){return t.useDocsData(e).versions},t.useLatestVersion=function(e){var n=t.useDocsData(e);return u.getLatestVersion(n)},t.useActiveVersion=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getActiveVersion(n,o)},t.useActiveDocContext=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getActiveDocContext(n,o)},t.useDocVersionSuggestions=function(e){var n=t.useDocsData(e),o=r.useLocation().pathname;return u.getDocVersionSuggestions(n,o)}},257:function(e,t,n){"use strict";n.r(t),n.d(t,"default",(function(){return o})),n.d(t,"useAllPluginInstancesData",(function(){return u})),n.d(t,"usePluginData",(function(){return i}));var r=n(16);function o(){var e=Object(r.default)().globalData;if(!e)throw new Error("Docusaurus global data not found");return e}function u(e){var t=o()[e];if(!t)throw new Error("Docusaurus plugin global data not found for pluginName="+e);return t}function i(e,t){void 0===t&&(t="default");var n=u(e)[t];if(!n)throw new Error("Docusaurus plugin global data not found for pluginName="+e+" and pluginId="+t);return n}},258:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getDocVersionSuggestions=t.getActiveDocContext=t.getActiveVersion=t.getLatestVersion=t.getActivePlugin=void 0;var r=n(23);t.getActivePlugin=function(e,t,n){void 0===n&&(n={});var o=Object.entries(e).find((function(e){e[0];var n=e[1];return!!r.matchPath(t,{path:n.path,exact:!1,strict:!1})})),u=o?{pluginId:o[0],pluginData:o[1]}:void 0;if(!u&&n.failfast)throw new Error("Can't find active docs plugin for pathname="+t+", while it was expected to be found. Maybe you tried to use a docs feature that can only be used on a docs-related page? Existing docs plugin paths are: "+Object.values(e).map((function(e){return e.path})).join(", "));return u},t.getLatestVersion=function(e){return e.versions.find((function(e){return e.isLast}))},t.getActiveVersion=function(e,n){var o=t.getLatestVersion(e);return[].concat(e.versions.filter((function(e){return e!==o})),[o]).find((function(e){return!!r.matchPath(n,{path:e.path,exact:!1,strict:!1})}))},t.getActiveDocContext=function(e,n){var o,u,i=t.getActiveVersion(e,n),a=null==i?void 0:i.docs.find((function(e){return!!r.matchPath(n,{path:e.path,exact:!0,strict:!1})}));return{activeVersion:i,activeDoc:a,alternateDocVersions:a?(o=a.id,u={},e.versions.forEach((function(e){e.docs.forEach((function(t){t.id===o&&(u[e.name]=t)}))})),u):{}}},t.getDocVersionSuggestions=function(e,n){var r=t.getLatestVersion(e),o=t.getActiveDocContext(e,n),u=o.activeVersion!==r;return{latestDocSuggestion:u?null==o?void 0:o.alternateDocVersions[r.name]:void 0,latestVersionSuggestion:u?r:void 0}}},259:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.isSamePath=void 0,t.isSamePath=function(e,t){var n=function(e){return!e||(null==e?void 0:e.endsWith("/"))?e:e+"/"};return n(e)===n(t)}},260:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.useTitleFormatter=void 0;var o=r(n(16));t.useTitleFormatter=function(e){var t=o.default().siteConfig,n=void 0===t?{}:t,r=n.title,u=n.titleDelimiter,i=void 0===u?"|":u;return e&&e.trim().length?e.trim()+" "+i+" "+r:r}},261:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.usePluralForm=void 0;var o=n(0),u=r(n(16)),i=["zero","one","two","few","many","other"];function a(e){return i.filter((function(t){return e.includes(t)}))}var s={locale:"en",pluralForms:a(["one","other"]),select:function(e){return 1===e?"one":"other"}};function c(){var e=u.default().i18n.currentLocale;return o.useMemo((function(){if(!Intl.PluralRules)return console.error("Intl.PluralRules not available!\nDocusaurus will fallback to a default/fallback (English) Intl.PluralRules implementation.\n        "),s;try{return t=e,n=new Intl.PluralRules(t),{locale:t,pluralForms:a(n.resolvedOptions().pluralCategories),select:function(e){return n.select(e)}}}catch(r){return console.error("Failed to use Intl.PluralRules for locale="+e+".\nDocusaurus will fallback to a default/fallback (English) Intl.PluralRules implementation.\n"),s}var t,n}),[e])}t.usePluralForm=function(){var e=c();return{selectMessage:function(t,n){return function(e,t,n){var r=e.split("|");if(1===r.length)return r[0];r.length>n.pluralForms.length&&console.error("For locale="+n.locale+", a maximum of "+n.pluralForms.length+" plural forms are expected ("+n.pluralForms+"), but the message contains "+r.length+" plural forms: "+e+" ");var o=n.select(t),u=n.pluralForms.indexOf(o);return r[Math.min(u,r.length-1)]}(n,t,e)}}}},262:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.useDocsPreferredVersionByPluginId=t.useDocsPreferredVersion=void 0;var r=n(0),o=n(242),u=n(232),i=n(264);t.useDocsPreferredVersion=function(e){void 0===e&&(e=i.DEFAULT_PLUGIN_ID);var t=u.useDocsData(e),n=o.useDocsPreferredVersionContext(),a=n[0],s=n[1],c=a[e].preferredVersionName;return{preferredVersion:c?t.versions.find((function(e){return e.name===c})):null,savePreferredVersionName:r.useCallback((function(t){s.savePreferredVersion(e,t)}),[s])}},t.useDocsPreferredVersionByPluginId=function(){var e=u.useAllDocsData(),t=o.useDocsPreferredVersionContext()[0],n=Object.keys(e),r={};return n.forEach((function(n){r[n]=function(n){var r=e[n],o=t[n].preferredVersionName;return o?r.versions.find((function(e){return e.name===o})):null}(n)})),r}},263:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=function(e){return"docs-preferred-version-"+e},o={save:function(e,t,n){"none"===t||window.localStorage.setItem(r(e),n)},read:function(e,t){return"none"===t?null:window.localStorage.getItem(r(e))},clear:function(e,t){"none"===t||window.localStorage.removeItem(r(e))}};t.default=o},264:function(e,t,n){"use strict";n.r(t),n.d(t,"DEFAULT_PLUGIN_ID",(function(){return r}));var r="default"}}]);
\ No newline at end of file
diff --git a/assets/js/418618b7.efff362b.js b/assets/js/12ba72b5.4b8388c3.js
similarity index 84%
rename from assets/js/418618b7.efff362b.js
rename to assets/js/12ba72b5.4b8388c3.js
index 577f16a6..b00d5097 100644
--- a/assets/js/418618b7.efff362b.js
+++ b/assets/js/12ba72b5.4b8388c3.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[33],{103:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-2.0.0/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-2.0.0/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_install_simple.md",version:"2.0.0",sidebar_label:"Docker Installation",sidebar:"version-2.0.0/Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[16],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},78:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-2.1.0/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-2.1.0/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_install_simple.md",version:"2.1.0",sidebar_label:"Docker Installation",sidebar:"version-2.1.0/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/pipeline/elastic"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/12ecfc4a.a1b9aa60.js b/assets/js/12ecfc4a.a1b9aa60.js
new file mode 100644
index 00000000..767b044e
--- /dev/null
+++ b/assets/js/12ecfc4a.a1b9aa60.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[17],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},79:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-2.1.0/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-2.1.0/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/pipeline_dataset.md",version:"2.1.0",sidebar_label:"Replay Dataset",sidebar:"version-2.1.0/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/161.d19b2b8e.js b/assets/js/157.1f338186.js
similarity index 95%
rename from assets/js/161.d19b2b8e.js
rename to assets/js/157.1f338186.js
index 06f39e4d..2bcde21f 100644
--- a/assets/js/161.d19b2b8e.js
+++ b/assets/js/157.1f338186.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[161,162],{230:function(e,t,a){"use strict";a.r(t);var n=a(0),r=a.n(n),o=a(232),c=a(16),l=a(27),i=a(240),s=a(3),u=a(7),d=a(234),p=a(233),m=a(247),b=a(252),h=a(253),f=a(251),y=a(237),g=a(242),v=a(254),j=function(e){return r.a.createElement("svg",Object(s.a)({width:"20",height:"20",role:"img"},e),r.a.createElement("g",{fill:"#7a7a7a"},r.a.createElement("path",{d:"M9.992 10.023c0 .2-.062.399-.172.547l-4.996 7.492a.982.982 0 01-.828.454H1c-.55 0-1-.453-1-1 0-.2.059-.403.168-.551l4.629-6.942L.168 3.078A.939.939 0 010 2.528c0-.548.45-.997 1-.997h2.996c.352 0 .649.18.828.45L9.82 9.472c.11.148.172.347.172.55zm0 0"}),r.a.createElement("path",{d:"M19.98 10.023c0 .2-.058.399-.168.547l-4.996 7.492a.987.987 0 01-.828.454h-3c-.547 0-.996-.453-.996-1 0-.2.059-.403.168-.551l4.625-6.942-4.625-6.945a.939.939 0 01-.168-.55 1 1 0 01.996-.997h3c.348 0 .649.18.828.45l4.996 7.492c.11.148.168.347.168.55zm0 0"})))},k=a(256),O=a(235),E=a(84),C=a.n(E);var N=function e(t,a){return"link"===t.type?Object(p.isSamePath)(t.href,a):"category"===t.type&&t.items.some((function(t){return e(t,a)}))};function T(e){var t,a,o,c=e.item,l=e.onItemClick,i=e.collapsible,p=e.activePath,m=Object(u.a)(e,["item","onItemClick","collapsible","activePath"]),b=c.items,h=c.label,f=N(c,p),y=(a=f,o=Object(n.useRef)(a),Object(n.useEffect)((function(){o.current=a}),[a]),o.current),g=Object(n.useState)((function(){return!!i&&(!f&&c.collapsed)})),v=g[0],j=g[1],k=Object(n.useRef)(null),O=Object(n.useState)(void 0),E=O[0],T=O[1],x=function(e){var t;void 0===e&&(e=!0),T(e?(null===(t=k.current)||void 0===t?void 0:t.scrollHeight)+"px":void 0)};Object(n.useEffect)((function(){f&&!y&&v&&j(!1)}),[f,y,v]);var P=Object(n.useCallback)((function(e){e.preventDefault(),E||x(),setTimeout((function(){return j((function(e){return!e}))}),100)}),[E]);return 0===b.length?null:r.a.createElement("li",{className:Object(d.a)("menu__list-item",{"menu__list-item--collapsed":v}),key:h},r.a.createElement("a",Object(s.a)({className:Object(d.a)("menu__link",(t={"menu__link--sublist":i,"menu__link--active":i&&f},t[C.a.menuLinkText]=!i,t)),onClick:i?P:void 0,href:i?"#!":void 0},m),h),r.a.createElement("ul",{className:"menu__list",ref:k,style:{height:E},onTransitionEnd:function(){v||x(!1)}},b.map((function(e){return r.a.createElement(w,{tabIndex:v?"-1":"0",key:e.label,item:e,onItemClick:l,collapsible:i,activePath:p})}))))}function x(e){var t,a=e.item,n=e.onItemClick,o=e.activePath,c=(e.collapsible,Object(u.a)(e,["item","onItemClick","activePath","collapsible"])),l=a.href,i=a.label,p=N(a,o);return r.a.createElement("li",{className:"menu__list-item",key:i},r.a.createElement(y.a,Object(s.a)({className:Object(d.a)("menu__link",(t={"menu__link--active":p},t[C.a.menuLinkExternal]=!Object(g.a)(l),t)),to:l},Object(g.a)(l)&&{isNavLink:!0,exact:!0,onClick:n},c),i))}function w(e){switch(e.item.type){case"category":return r.a.createElement(T,e);case"link":default:return r.a.createElement(x,e)}}var P=function(e){var t,a,o=e.path,c=e.sidebar,l=e.sidebarCollapsible,i=void 0===l||l,s=e.onCollapse,u=e.isHidden,y=Object(n.useState)(!1),g=y[0],E=y[1],N=Object(p.useThemeConfig)(),T=N.navbar.hideOnScroll,x=N.hideableSidebar,P=Object(m.a)().isAnnouncementBarClosed,S=Object(f.a)().scrollY;Object(b.a)(g);var I=Object(h.a)();return Object(n.useEffect)((function(){I===h.b.desktop&&E(!1)}),[I]),r.a.createElement("div",{className:Object(d.a)(C.a.sidebar,(t={},t[C.a.sidebarWithHideableNavbar]=T,t[C.a.sidebarHidden]=u,t))},T&&r.a.createElement(v.a,{tabIndex:-1,className:C.a.sidebarLogo}),r.a.createElement("div",{className:Object(d.a)("menu","menu--responsive","thin-scrollbar",C.a.menu,(a={"menu--show":g},a[C.a.menuWithAnnouncementBar]=!P&&0===S,a))},r.a.createElement("button",{"aria-label":g?Object(O.b)({id:"theme.docs.sidebar.responsiveCloseButtonLabel",message:"Close menu",description:"The ARIA label for close button of mobile doc sidebar"}):Object(O.b)({id:"theme.docs.sidebar.responsiveOpenButtonLabel",message:"Open menu",description:"The ARIA label for open button of mobile doc sidebar"}),"aria-haspopup":"true",className:"button button--secondary button--sm menu__button",type:"button",onClick:function(){E(!g)}},g?r.a.createElement("span",{className:Object(d.a)(C.a.sidebarMenuIcon,C.a.sidebarMenuCloseIcon)},"\xd7"):r.a.createElement(k.a,{className:C.a.sidebarMenuIcon,height:24,width:24})),r.a.createElement("ul",{className:"menu__list"},c.map((function(e){return r.a.createElement(w,{key:e.label,item:e,onItemClick:function(e){e.target.blur(),E(!1)},collapsible:i,activePath:o})})))),x&&r.a.createElement("button",{type:"button",title:Object(O.b)({id:"theme.docs.sidebar.collapseButtonTitle",message:"Collapse sidebar",description:"The title attribute for collapse button of doc sidebar"}),"aria-label":Object(O.b)({id:"theme.docs.sidebar.collapseButtonAriaLabel",message:"Collapse sidebar",description:"The title attribute for collapse button of doc sidebar"}),className:Object(d.a)("button button--secondary button--outline",C.a.collapseSidebarButton),onClick:s},r.a.createElement(j,{className:C.a.collapseSidebarButtonIcon})))},S={plain:{backgroundColor:"#2a2734",color:"#9a86fd"},styles:[{types:["comment","prolog","doctype","cdata","punctuation"],style:{color:"#6c6783"}},{types:["namespace"],style:{opacity:.7}},{types:["tag","operator","number"],style:{color:"#e09142"}},{types:["property","function"],style:{color:"#9a86fd"}},{types:["tag-id","selector","atrule-id"],style:{color:"#eeebff"}},{types:["attr-name"],style:{color:"#c4b9fe"}},{types:["boolean","string","entity","url","attr-value","keyword","control","directive","unit","statement","regex","at-rule","placeholder","variable"],style:{color:"#ffcc99"}},{types:["deleted"],style:{textDecorationLine:"line-through"}},{types:["inserted"],style:{textDecorationLine:"underline"}},{types:["italic"],style:{fontStyle:"italic"}},{types:["important","bold"],style:{fontWeight:"bold"}},{types:["important"],style:{color:"#c4b9fe"}}]},I={Prism:a(24).a,theme:S};function _(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function B(){return(B=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var a=arguments[t];for(var n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n])}return e}).apply(this,arguments)}var L=/\r\n|\r|\n/,A=function(e){0===e.length?e.push({types:["plain"],content:"\n",empty:!0}):1===e.length&&""===e[0].content&&(e[0].content="\n",e[0].empty=!0)},D=function(e,t){var a=e.length;return a>0&&e[a-1]===t?e:e.concat(t)},R=function(e,t){var a=e.plain,n=Object.create(null),r=e.styles.reduce((function(e,a){var n=a.languages,r=a.style;return n&&!n.includes(t)||a.types.forEach((function(t){var a=B({},e[t],r);e[t]=a})),e}),n);return r.root=a,r.plain=B({},a,{backgroundColor:null}),r};function M(e,t){var a={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&-1===t.indexOf(n)&&(a[n]=e[n]);return a}var F=function(e){function t(){for(var t=this,a=[],n=arguments.length;n--;)a[n]=arguments[n];e.apply(this,a),_(this,"getThemeDict",(function(e){if(void 0!==t.themeDict&&e.theme===t.prevTheme&&e.language===t.prevLanguage)return t.themeDict;t.prevTheme=e.theme,t.prevLanguage=e.language;var a=e.theme?R(e.theme,e.language):void 0;return t.themeDict=a})),_(this,"getLineProps",(function(e){var a=e.key,n=e.className,r=e.style,o=B({},M(e,["key","className","style","line"]),{className:"token-line",style:void 0,key:void 0}),c=t.getThemeDict(t.props);return void 0!==c&&(o.style=c.plain),void 0!==r&&(o.style=void 0!==o.style?B({},o.style,r):r),void 0!==a&&(o.key=a),n&&(o.className+=" "+n),o})),_(this,"getStyleForToken",(function(e){var a=e.types,n=e.empty,r=a.length,o=t.getThemeDict(t.props);if(void 0!==o){if(1===r&&"plain"===a[0])return n?{display:"inline-block"}:void 0;if(1===r&&!n)return o[a[0]];var c=n?{display:"inline-block"}:{},l=a.map((function(e){return o[e]}));return Object.assign.apply(Object,[c].concat(l))}})),_(this,"getTokenProps",(function(e){var a=e.key,n=e.className,r=e.style,o=e.token,c=B({},M(e,["key","className","style","token"]),{className:"token "+o.types.join(" "),children:o.content,style:t.getStyleForToken(o),key:void 0});return void 0!==r&&(c.style=void 0!==c.style?B({},c.style,r):r),void 0!==a&&(c.key=a),n&&(c.className+=" "+n),c})),_(this,"tokenize",(function(e,t,a,n){var r={code:t,grammar:a,language:n,tokens:[]};e.hooks.run("before-tokenize",r);var o=r.tokens=e.tokenize(r.code,r.grammar,r.language);return e.hooks.run("after-tokenize",r),o}))}return e&&(t.__proto__=e),t.prototype=Object.create(e&&e.prototype),t.prototype.constructor=t,t.prototype.render=function(){var e=this.props,t=e.Prism,a=e.language,n=e.code,r=e.children,o=this.getThemeDict(this.props),c=t.languages[a];return r({tokens:function(e){for(var t=[[]],a=[e],n=[0],r=[e.length],o=0,c=0,l=[],i=[l];c>-1;){for(;(o=n[c]++)<r[c];){var s=void 0,u=t[c],d=a[c][o];if("string"==typeof d?(u=c>0?u:["plain"],s=d):(u=D(u,d.type),d.alias&&(u=D(u,d.alias)),s=d.content),"string"==typeof s){var p=s.split(L),m=p.length;l.push({types:u,content:p[0]});for(var b=1;b<m;b++)A(l),i.push(l=[]),l.push({types:u,content:p[b]})}else c++,t.push(u),a.push(s),n.push(0),r.push(s.length)}c--,t.pop(),a.pop(),n.pop(),r.pop()}return A(l),i}(void 0!==c?this.tokenize(t,n,c,a):[n]),className:"prism-code language-"+a,style:void 0!==o?o.root:{},getLineProps:this.getLineProps,getTokenProps:this.getTokenProps})},t}(n.Component);var z=a(276),H=a.n(z),W={plain:{color:"#bfc7d5",backgroundColor:"#292d3e"},styles:[{types:["comment"],style:{color:"rgb(105, 112, 152)",fontStyle:"italic"}},{types:["string","inserted"],style:{color:"rgb(195, 232, 141)"}},{types:["number"],style:{color:"rgb(247, 140, 108)"}},{types:["builtin","char","constant","function"],style:{color:"rgb(130, 170, 255)"}},{types:["punctuation","selector"],style:{color:"rgb(199, 146, 234)"}},{types:["variable"],style:{color:"rgb(191, 199, 213)"}},{types:["class-name","attr-name"],style:{color:"rgb(255, 203, 107)"}},{types:["tag","deleted"],style:{color:"rgb(255, 85, 114)"}},{types:["operator"],style:{color:"rgb(137, 221, 255)"}},{types:["boolean"],style:{color:"rgb(255, 88, 116)"}},{types:["keyword"],style:{fontStyle:"italic"}},{types:["doctype"],style:{color:"rgb(199, 146, 234)",fontStyle:"italic"}},{types:["namespace"],style:{color:"rgb(178, 204, 214)"}},{types:["url"],style:{color:"rgb(221, 221, 221)"}}]},$=a(241),J=function(){var e=Object(p.useThemeConfig)().prism,t=Object($.a)().isDarkTheme,a=e.theme||W,n=e.darkTheme||a;return t?n:a},V=a(85),K=a.n(V),U=/{([\d,-]+)}/,X=function(e){void 0===e&&(e=["js","jsBlock","jsx","python","html"]);var t={js:{start:"\\/\\/",end:""},jsBlock:{start:"\\/\\*",end:"\\*\\/"},jsx:{start:"\\{\\s*\\/\\*",end:"\\*\\/\\s*\\}"},python:{start:"#",end:""},html:{start:"\x3c!--",end:"--\x3e"}},a=["highlight-next-line","highlight-start","highlight-end"].join("|"),n=e.map((function(e){return"(?:"+t[e].start+"\\s*("+a+")\\s*"+t[e].end+")"})).join("|");return new RegExp("^\\s*(?:"+n+")\\s*$")},Y=/(?:title=")(.*)(?:")/;function q(e){var t=e.children,a=e.className,o=e.metastring,c=Object(p.useThemeConfig)().prism,l=Object(n.useState)(!1),i=l[0],u=l[1],m=Object(n.useState)(!1),b=m[0],h=m[1];Object(n.useEffect)((function(){h(!0)}),[]);var f=Object(n.useRef)(null),y=[],g="",v=J(),j=Array.isArray(t)?t.join(""):t;if(o&&U.test(o)){var k=o.match(U)[1];y=H()(k).filter((function(e){return e>0}))}o&&Y.test(o)&&(g=o.match(Y)[1]);var E=a&&a.replace(/language-/,"");!E&&c.defaultLanguage&&(E=c.defaultLanguage);var C=j.replace(/\n$/,"");if(0===y.length&&void 0!==E){for(var N,T="",x=function(e){switch(e){case"js":case"javascript":case"ts":case"typescript":return X(["js","jsBlock"]);case"jsx":case"tsx":return X(["js","jsBlock","jsx"]);case"html":return X(["js","jsBlock","html"]);case"python":case"py":return X(["python"]);default:return X()}}(E),w=j.replace(/\n$/,"").split("\n"),P=0;P<w.length;){var S=P+1,_=w[P].match(x);if(null!==_){switch(_.slice(1).reduce((function(e,t){return e||t}),void 0)){case"highlight-next-line":T+=S+",";break;case"highlight-start":N=S;break;case"highlight-end":T+=N+"-"+(S-1)+","}w.splice(P,1)}else P+=1}y=H()(T),C=w.join("\n")}var B=function(){!function(e,{target:t=document.body}={}){const a=document.createElement("textarea"),n=document.activeElement;a.value=e,a.setAttribute("readonly",""),a.style.contain="strict",a.style.position="absolute",a.style.left="-9999px",a.style.fontSize="12pt";const r=document.getSelection();let o=!1;r.rangeCount>0&&(o=r.getRangeAt(0)),t.append(a),a.select(),a.selectionStart=0,a.selectionEnd=e.length;let c=!1;try{c=document.execCommand("copy")}catch{}a.remove(),o&&(r.removeAllRanges(),r.addRange(o)),n&&n.focus()}(C),u(!0),setTimeout((function(){return u(!1)}),2e3)};return r.a.createElement(F,Object(s.a)({},I,{key:String(b),theme:v,code:C,language:E}),(function(e){var t,a=e.className,n=e.style,o=e.tokens,c=e.getLineProps,l=e.getTokenProps;return r.a.createElement("div",{className:K.a.codeBlockContainer},g&&r.a.createElement("div",{style:n,className:K.a.codeBlockTitle},g),r.a.createElement("div",{className:Object(d.a)(K.a.codeBlockContent,E)},r.a.createElement("div",{tabIndex:0,className:Object(d.a)(a,K.a.codeBlock,"thin-scrollbar",(t={},t[K.a.codeBlockWithTitle]=g,t))},r.a.createElement("div",{className:K.a.codeBlockLines,style:n},o.map((function(e,t){1===e.length&&""===e[0].content&&(e[0].content="\n");var a=c({line:e,key:t});return y.includes(t+1)&&(a.className=a.className+" docusaurus-highlight-code-line"),r.a.createElement("div",Object(s.a)({key:t},a),e.map((function(e,t){return r.a.createElement("span",Object(s.a)({key:t},l({token:e,key:t})))})))})))),r.a.createElement("button",{ref:f,type:"button","aria-label":Object(O.b)({id:"theme.CodeBlock.copyButtonAriaLabel",message:"Copy code to clipboard",description:"The ARIA label for copy code blocks button"}),className:Object(d.a)(K.a.copyButton),onClick:B},i?r.a.createElement(O.a,{id:"theme.CodeBlock.copied",description:"The copied button label on code blocks"},"Copied"):r.a.createElement(O.a,{id:"theme.CodeBlock.copy",description:"The copy button label on code blocks"},"Copy"))))}))}a(86);var G=a(87),Q=a.n(G),Z=function(e){return function(t){var a,n=t.id,o=Object(u.a)(t,["id"]),c=Object(p.useThemeConfig)().navbar.hideOnScroll;return n?r.a.createElement(e,o,r.a.createElement("a",{"aria-hidden":"true",tabIndex:-1,className:Object(d.a)("anchor",(a={},a[Q.a.enhancedAnchor]=!c,a)),id:n}),o.children,r.a.createElement("a",{className:"hash-link",href:"#"+n,title:Object(O.b)({id:"theme.common.headingLinkTitle",message:"Direct link to heading",description:"Title for link to heading"})},"#")):r.a.createElement(e,o)}},ee={code:function(e){var t=e.children;return"string"==typeof t?t.includes("\n")?r.a.createElement(q,e):r.a.createElement("code",e):t},a:function(e){return r.a.createElement(y.a,e)},pre:function(e){var t=e.children;return r.a.createElement(q,Object(n.isValidElement)(t)?null==t?void 0:t.props:{children:t})},h1:Z("h1"),h2:Z("h2"),h3:Z("h3"),h4:Z("h4"),h5:Z("h5"),h6:Z("h6")},te=a(243),ae=a(23),ne=a(88),re=a.n(ne);function oe(e){var t,a,l,s,u,m=e.currentDocRoute,b=e.versionMetadata,h=e.children,f=Object(c.default)(),y=f.siteConfig,g=f.isClient,v=b.pluginId,k=b.permalinkToSidebar,E=b.docsSidebars,C=b.version,N=k[m.path],T=E[N],x=Object(n.useState)(!1),w=x[0],S=x[1],I=Object(n.useState)(!1),_=I[0],B=I[1],L=Object(n.useCallback)((function(){_&&B(!1),S(!w)}),[_]);return r.a.createElement(i.a,{key:g,wrapperClassName:"main-docs-wrapper",searchMetadatas:{version:C,tag:Object(p.docVersionSearchTag)(v,C)}},r.a.createElement("div",{className:re.a.docPage},T&&r.a.createElement("div",{className:Object(d.a)(re.a.docSidebarContainer,(t={},t[re.a.docSidebarContainerHidden]=w,t)),onTransitionEnd:function(e){e.currentTarget.classList.contains(re.a.docSidebarContainer)&&w&&B(!0)},role:"complementary"},r.a.createElement(P,{key:N,sidebar:T,path:m.path,sidebarCollapsible:null===(a=null===(l=y.themeConfig)||void 0===l?void 0:l.sidebarCollapsible)||void 0===a||a,onCollapse:L,isHidden:_}),_&&r.a.createElement("div",{className:re.a.collapsedDocSidebar,title:Object(O.b)({id:"theme.docs.sidebar.expandButtonTitle",message:"Expand sidebar",description:"The ARIA label and title attribute for expand button of doc sidebar"}),"aria-label":Object(O.b)({id:"theme.docs.sidebar.expandButtonAriaLabel",message:"Expand sidebar",description:"The ARIA label and title attribute for expand button of doc sidebar"}),tabIndex:0,role:"button",onKeyDown:L,onClick:L},r.a.createElement(j,{className:re.a.expandSidebarButtonIcon}))),r.a.createElement("main",{className:Object(d.a)(re.a.docMainContainer,(s={},s[re.a.docMainContainerEnhanced]=w,s))},r.a.createElement("div",{className:Object(d.a)("container padding-vert--lg",re.a.docItemWrapper,(u={},u[re.a.docItemWrapperEnhanced]=w,u))},r.a.createElement(o.a,{components:ee},h)))))}t.default=function(e){var t=e.route.routes,a=e.versionMetadata,n=e.location,o=t.find((function(e){return Object(ae.matchPath)(n.pathname,e)}));return o?r.a.createElement(oe,{currentDocRoute:o,versionMetadata:a},Object(l.a)(t)):r.a.createElement(te.default,e)}},232:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return b}));var n=a(0),r=a.n(n);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function c(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?c(Object(a),!0).forEach((function(t){o(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):c(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function i(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},o=Object.keys(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=r.a.createContext({}),u=function(e){var t=r.a.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},d=function(e){var t=u(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,c=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),d=u(a),m=n,b=d["".concat(c,".").concat(m)]||d[m]||p[m]||o;return a?r.a.createElement(b,l(l({ref:t},s),{},{components:a})):r.a.createElement(b,l({ref:t},s))}));function b(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,c=new Array(o);c[0]=m;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l.mdxType="string"==typeof e?e:n,c[1]=l;for(var s=2;s<o;s++)c[s]=a[s];return r.a.createElement.apply(null,c)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"},243:function(e,t,a){"use strict";a.r(t);var n=a(0),r=a.n(n),o=a(240),c=a(235);t.default=function(){return r.a.createElement(o.a,{title:"Page Not Found"},r.a.createElement("main",{className:"container margin-vert--xl"},r.a.createElement("div",{className:"row"},r.a.createElement("div",{className:"col col--6 col--offset-3"},r.a.createElement("h1",{className:"hero__title"},r.a.createElement(c.a,{id:"theme.NotFound.title",description:"The title of the 404 page"},"Page Not Found")),r.a.createElement("p",null,r.a.createElement(c.a,{id:"theme.NotFound.p1",description:"The first paragraph of the 404 page"},"We could not find what you were looking for.")),r.a.createElement("p",null,r.a.createElement(c.a,{id:"theme.NotFound.p2",description:"The 2nd paragraph of the 404 page"},"Please contact the owner of the site that linked you to the original URL and let them know their link is broken."))))))}},276:function(e,t){function a(e){let t,a=[];for(let n of e.split(",").map((e=>e.trim())))if(/^-?\d+$/.test(n))a.push(parseInt(n,10));else if(t=n.match(/^(-?\d+)(-|\.\.\.?|\u2025|\u2026|\u22EF)(-?\d+)$/)){let[e,n,r,o]=t;if(n&&o){n=parseInt(n),o=parseInt(o);const e=n<o?1:-1;"-"!==r&&".."!==r&&"\u2025"!==r||(o+=e);for(let t=n;t!==o;t+=e)a.push(t)}}return a}t.default=a,e.exports=a}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[157,158],{226:function(e,t,a){"use strict";a.r(t);var n=a(0),r=a.n(n),o=a(228),c=a(16),l=a(27),i=a(236),s=a(3),u=a(7),d=a(230),p=a(229),m=a(243),b=a(248),h=a(249),f=a(247),y=a(233),g=a(238),v=a(250),j=function(e){return r.a.createElement("svg",Object(s.a)({width:"20",height:"20",role:"img"},e),r.a.createElement("g",{fill:"#7a7a7a"},r.a.createElement("path",{d:"M9.992 10.023c0 .2-.062.399-.172.547l-4.996 7.492a.982.982 0 01-.828.454H1c-.55 0-1-.453-1-1 0-.2.059-.403.168-.551l4.629-6.942L.168 3.078A.939.939 0 010 2.528c0-.548.45-.997 1-.997h2.996c.352 0 .649.18.828.45L9.82 9.472c.11.148.172.347.172.55zm0 0"}),r.a.createElement("path",{d:"M19.98 10.023c0 .2-.058.399-.168.547l-4.996 7.492a.987.987 0 01-.828.454h-3c-.547 0-.996-.453-.996-1 0-.2.059-.403.168-.551l4.625-6.942-4.625-6.945a.939.939 0 01-.168-.55 1 1 0 01.996-.997h3c.348 0 .649.18.828.45l4.996 7.492c.11.148.168.347.168.55zm0 0"})))},k=a(252),O=a(231),E=a(85),C=a.n(E);var N=function e(t,a){return"link"===t.type?Object(p.isSamePath)(t.href,a):"category"===t.type&&t.items.some((function(t){return e(t,a)}))};function T(e){var t,a,o,c=e.item,l=e.onItemClick,i=e.collapsible,p=e.activePath,m=Object(u.a)(e,["item","onItemClick","collapsible","activePath"]),b=c.items,h=c.label,f=N(c,p),y=(a=f,o=Object(n.useRef)(a),Object(n.useEffect)((function(){o.current=a}),[a]),o.current),g=Object(n.useState)((function(){return!!i&&(!f&&c.collapsed)})),v=g[0],j=g[1],k=Object(n.useRef)(null),O=Object(n.useState)(void 0),E=O[0],T=O[1],x=function(e){var t;void 0===e&&(e=!0),T(e?(null===(t=k.current)||void 0===t?void 0:t.scrollHeight)+"px":void 0)};Object(n.useEffect)((function(){f&&!y&&v&&j(!1)}),[f,y,v]);var P=Object(n.useCallback)((function(e){e.preventDefault(),E||x(),setTimeout((function(){return j((function(e){return!e}))}),100)}),[E]);return 0===b.length?null:r.a.createElement("li",{className:Object(d.a)("menu__list-item",{"menu__list-item--collapsed":v}),key:h},r.a.createElement("a",Object(s.a)({className:Object(d.a)("menu__link",(t={"menu__link--sublist":i,"menu__link--active":i&&f},t[C.a.menuLinkText]=!i,t)),onClick:i?P:void 0,href:i?"#!":void 0},m),h),r.a.createElement("ul",{className:"menu__list",ref:k,style:{height:E},onTransitionEnd:function(){v||x(!1)}},b.map((function(e){return r.a.createElement(w,{tabIndex:v?"-1":"0",key:e.label,item:e,onItemClick:l,collapsible:i,activePath:p})}))))}function x(e){var t,a=e.item,n=e.onItemClick,o=e.activePath,c=(e.collapsible,Object(u.a)(e,["item","onItemClick","activePath","collapsible"])),l=a.href,i=a.label,p=N(a,o);return r.a.createElement("li",{className:"menu__list-item",key:i},r.a.createElement(y.a,Object(s.a)({className:Object(d.a)("menu__link",(t={"menu__link--active":p},t[C.a.menuLinkExternal]=!Object(g.a)(l),t)),to:l},Object(g.a)(l)&&{isNavLink:!0,exact:!0,onClick:n},c),i))}function w(e){switch(e.item.type){case"category":return r.a.createElement(T,e);case"link":default:return r.a.createElement(x,e)}}var P=function(e){var t,a,o=e.path,c=e.sidebar,l=e.sidebarCollapsible,i=void 0===l||l,s=e.onCollapse,u=e.isHidden,y=Object(n.useState)(!1),g=y[0],E=y[1],N=Object(p.useThemeConfig)(),T=N.navbar.hideOnScroll,x=N.hideableSidebar,P=Object(m.a)().isAnnouncementBarClosed,S=Object(f.a)().scrollY;Object(b.a)(g);var I=Object(h.a)();return Object(n.useEffect)((function(){I===h.b.desktop&&E(!1)}),[I]),r.a.createElement("div",{className:Object(d.a)(C.a.sidebar,(t={},t[C.a.sidebarWithHideableNavbar]=T,t[C.a.sidebarHidden]=u,t))},T&&r.a.createElement(v.a,{tabIndex:-1,className:C.a.sidebarLogo}),r.a.createElement("div",{className:Object(d.a)("menu","menu--responsive","thin-scrollbar",C.a.menu,(a={"menu--show":g},a[C.a.menuWithAnnouncementBar]=!P&&0===S,a))},r.a.createElement("button",{"aria-label":g?Object(O.b)({id:"theme.docs.sidebar.responsiveCloseButtonLabel",message:"Close menu",description:"The ARIA label for close button of mobile doc sidebar"}):Object(O.b)({id:"theme.docs.sidebar.responsiveOpenButtonLabel",message:"Open menu",description:"The ARIA label for open button of mobile doc sidebar"}),"aria-haspopup":"true",className:"button button--secondary button--sm menu__button",type:"button",onClick:function(){E(!g)}},g?r.a.createElement("span",{className:Object(d.a)(C.a.sidebarMenuIcon,C.a.sidebarMenuCloseIcon)},"\xd7"):r.a.createElement(k.a,{className:C.a.sidebarMenuIcon,height:24,width:24})),r.a.createElement("ul",{className:"menu__list"},c.map((function(e){return r.a.createElement(w,{key:e.label,item:e,onItemClick:function(e){e.target.blur(),E(!1)},collapsible:i,activePath:o})})))),x&&r.a.createElement("button",{type:"button",title:Object(O.b)({id:"theme.docs.sidebar.collapseButtonTitle",message:"Collapse sidebar",description:"The title attribute for collapse button of doc sidebar"}),"aria-label":Object(O.b)({id:"theme.docs.sidebar.collapseButtonAriaLabel",message:"Collapse sidebar",description:"The title attribute for collapse button of doc sidebar"}),className:Object(d.a)("button button--secondary button--outline",C.a.collapseSidebarButton),onClick:s},r.a.createElement(j,{className:C.a.collapseSidebarButtonIcon})))},S={plain:{backgroundColor:"#2a2734",color:"#9a86fd"},styles:[{types:["comment","prolog","doctype","cdata","punctuation"],style:{color:"#6c6783"}},{types:["namespace"],style:{opacity:.7}},{types:["tag","operator","number"],style:{color:"#e09142"}},{types:["property","function"],style:{color:"#9a86fd"}},{types:["tag-id","selector","atrule-id"],style:{color:"#eeebff"}},{types:["attr-name"],style:{color:"#c4b9fe"}},{types:["boolean","string","entity","url","attr-value","keyword","control","directive","unit","statement","regex","at-rule","placeholder","variable"],style:{color:"#ffcc99"}},{types:["deleted"],style:{textDecorationLine:"line-through"}},{types:["inserted"],style:{textDecorationLine:"underline"}},{types:["italic"],style:{fontStyle:"italic"}},{types:["important","bold"],style:{fontWeight:"bold"}},{types:["important"],style:{color:"#c4b9fe"}}]},I={Prism:a(24).a,theme:S};function _(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function B(){return(B=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var a=arguments[t];for(var n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n])}return e}).apply(this,arguments)}var L=/\r\n|\r|\n/,A=function(e){0===e.length?e.push({types:["plain"],content:"\n",empty:!0}):1===e.length&&""===e[0].content&&(e[0].content="\n",e[0].empty=!0)},D=function(e,t){var a=e.length;return a>0&&e[a-1]===t?e:e.concat(t)},R=function(e,t){var a=e.plain,n=Object.create(null),r=e.styles.reduce((function(e,a){var n=a.languages,r=a.style;return n&&!n.includes(t)||a.types.forEach((function(t){var a=B({},e[t],r);e[t]=a})),e}),n);return r.root=a,r.plain=B({},a,{backgroundColor:null}),r};function M(e,t){var a={};for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&-1===t.indexOf(n)&&(a[n]=e[n]);return a}var F=function(e){function t(){for(var t=this,a=[],n=arguments.length;n--;)a[n]=arguments[n];e.apply(this,a),_(this,"getThemeDict",(function(e){if(void 0!==t.themeDict&&e.theme===t.prevTheme&&e.language===t.prevLanguage)return t.themeDict;t.prevTheme=e.theme,t.prevLanguage=e.language;var a=e.theme?R(e.theme,e.language):void 0;return t.themeDict=a})),_(this,"getLineProps",(function(e){var a=e.key,n=e.className,r=e.style,o=B({},M(e,["key","className","style","line"]),{className:"token-line",style:void 0,key:void 0}),c=t.getThemeDict(t.props);return void 0!==c&&(o.style=c.plain),void 0!==r&&(o.style=void 0!==o.style?B({},o.style,r):r),void 0!==a&&(o.key=a),n&&(o.className+=" "+n),o})),_(this,"getStyleForToken",(function(e){var a=e.types,n=e.empty,r=a.length,o=t.getThemeDict(t.props);if(void 0!==o){if(1===r&&"plain"===a[0])return n?{display:"inline-block"}:void 0;if(1===r&&!n)return o[a[0]];var c=n?{display:"inline-block"}:{},l=a.map((function(e){return o[e]}));return Object.assign.apply(Object,[c].concat(l))}})),_(this,"getTokenProps",(function(e){var a=e.key,n=e.className,r=e.style,o=e.token,c=B({},M(e,["key","className","style","token"]),{className:"token "+o.types.join(" "),children:o.content,style:t.getStyleForToken(o),key:void 0});return void 0!==r&&(c.style=void 0!==c.style?B({},c.style,r):r),void 0!==a&&(c.key=a),n&&(c.className+=" "+n),c})),_(this,"tokenize",(function(e,t,a,n){var r={code:t,grammar:a,language:n,tokens:[]};e.hooks.run("before-tokenize",r);var o=r.tokens=e.tokenize(r.code,r.grammar,r.language);return e.hooks.run("after-tokenize",r),o}))}return e&&(t.__proto__=e),t.prototype=Object.create(e&&e.prototype),t.prototype.constructor=t,t.prototype.render=function(){var e=this.props,t=e.Prism,a=e.language,n=e.code,r=e.children,o=this.getThemeDict(this.props),c=t.languages[a];return r({tokens:function(e){for(var t=[[]],a=[e],n=[0],r=[e.length],o=0,c=0,l=[],i=[l];c>-1;){for(;(o=n[c]++)<r[c];){var s=void 0,u=t[c],d=a[c][o];if("string"==typeof d?(u=c>0?u:["plain"],s=d):(u=D(u,d.type),d.alias&&(u=D(u,d.alias)),s=d.content),"string"==typeof s){var p=s.split(L),m=p.length;l.push({types:u,content:p[0]});for(var b=1;b<m;b++)A(l),i.push(l=[]),l.push({types:u,content:p[b]})}else c++,t.push(u),a.push(s),n.push(0),r.push(s.length)}c--,t.pop(),a.pop(),n.pop(),r.pop()}return A(l),i}(void 0!==c?this.tokenize(t,n,c,a):[n]),className:"prism-code language-"+a,style:void 0!==o?o.root:{},getLineProps:this.getLineProps,getTokenProps:this.getTokenProps})},t}(n.Component);var z=a(272),H=a.n(z),W={plain:{color:"#bfc7d5",backgroundColor:"#292d3e"},styles:[{types:["comment"],style:{color:"rgb(105, 112, 152)",fontStyle:"italic"}},{types:["string","inserted"],style:{color:"rgb(195, 232, 141)"}},{types:["number"],style:{color:"rgb(247, 140, 108)"}},{types:["builtin","char","constant","function"],style:{color:"rgb(130, 170, 255)"}},{types:["punctuation","selector"],style:{color:"rgb(199, 146, 234)"}},{types:["variable"],style:{color:"rgb(191, 199, 213)"}},{types:["class-name","attr-name"],style:{color:"rgb(255, 203, 107)"}},{types:["tag","deleted"],style:{color:"rgb(255, 85, 114)"}},{types:["operator"],style:{color:"rgb(137, 221, 255)"}},{types:["boolean"],style:{color:"rgb(255, 88, 116)"}},{types:["keyword"],style:{fontStyle:"italic"}},{types:["doctype"],style:{color:"rgb(199, 146, 234)",fontStyle:"italic"}},{types:["namespace"],style:{color:"rgb(178, 204, 214)"}},{types:["url"],style:{color:"rgb(221, 221, 221)"}}]},$=a(237),J=function(){var e=Object(p.useThemeConfig)().prism,t=Object($.a)().isDarkTheme,a=e.theme||W,n=e.darkTheme||a;return t?n:a},V=a(86),K=a.n(V),U=/{([\d,-]+)}/,X=function(e){void 0===e&&(e=["js","jsBlock","jsx","python","html"]);var t={js:{start:"\\/\\/",end:""},jsBlock:{start:"\\/\\*",end:"\\*\\/"},jsx:{start:"\\{\\s*\\/\\*",end:"\\*\\/\\s*\\}"},python:{start:"#",end:""},html:{start:"\x3c!--",end:"--\x3e"}},a=["highlight-next-line","highlight-start","highlight-end"].join("|"),n=e.map((function(e){return"(?:"+t[e].start+"\\s*("+a+")\\s*"+t[e].end+")"})).join("|");return new RegExp("^\\s*(?:"+n+")\\s*$")},Y=/(?:title=")(.*)(?:")/;function q(e){var t=e.children,a=e.className,o=e.metastring,c=Object(p.useThemeConfig)().prism,l=Object(n.useState)(!1),i=l[0],u=l[1],m=Object(n.useState)(!1),b=m[0],h=m[1];Object(n.useEffect)((function(){h(!0)}),[]);var f=Object(n.useRef)(null),y=[],g="",v=J(),j=Array.isArray(t)?t.join(""):t;if(o&&U.test(o)){var k=o.match(U)[1];y=H()(k).filter((function(e){return e>0}))}o&&Y.test(o)&&(g=o.match(Y)[1]);var E=a&&a.replace(/language-/,"");!E&&c.defaultLanguage&&(E=c.defaultLanguage);var C=j.replace(/\n$/,"");if(0===y.length&&void 0!==E){for(var N,T="",x=function(e){switch(e){case"js":case"javascript":case"ts":case"typescript":return X(["js","jsBlock"]);case"jsx":case"tsx":return X(["js","jsBlock","jsx"]);case"html":return X(["js","jsBlock","html"]);case"python":case"py":return X(["python"]);default:return X()}}(E),w=j.replace(/\n$/,"").split("\n"),P=0;P<w.length;){var S=P+1,_=w[P].match(x);if(null!==_){switch(_.slice(1).reduce((function(e,t){return e||t}),void 0)){case"highlight-next-line":T+=S+",";break;case"highlight-start":N=S;break;case"highlight-end":T+=N+"-"+(S-1)+","}w.splice(P,1)}else P+=1}y=H()(T),C=w.join("\n")}var B=function(){!function(e,{target:t=document.body}={}){const a=document.createElement("textarea"),n=document.activeElement;a.value=e,a.setAttribute("readonly",""),a.style.contain="strict",a.style.position="absolute",a.style.left="-9999px",a.style.fontSize="12pt";const r=document.getSelection();let o=!1;r.rangeCount>0&&(o=r.getRangeAt(0)),t.append(a),a.select(),a.selectionStart=0,a.selectionEnd=e.length;let c=!1;try{c=document.execCommand("copy")}catch{}a.remove(),o&&(r.removeAllRanges(),r.addRange(o)),n&&n.focus()}(C),u(!0),setTimeout((function(){return u(!1)}),2e3)};return r.a.createElement(F,Object(s.a)({},I,{key:String(b),theme:v,code:C,language:E}),(function(e){var t,a=e.className,n=e.style,o=e.tokens,c=e.getLineProps,l=e.getTokenProps;return r.a.createElement("div",{className:K.a.codeBlockContainer},g&&r.a.createElement("div",{style:n,className:K.a.codeBlockTitle},g),r.a.createElement("div",{className:Object(d.a)(K.a.codeBlockContent,E)},r.a.createElement("div",{tabIndex:0,className:Object(d.a)(a,K.a.codeBlock,"thin-scrollbar",(t={},t[K.a.codeBlockWithTitle]=g,t))},r.a.createElement("div",{className:K.a.codeBlockLines,style:n},o.map((function(e,t){1===e.length&&""===e[0].content&&(e[0].content="\n");var a=c({line:e,key:t});return y.includes(t+1)&&(a.className=a.className+" docusaurus-highlight-code-line"),r.a.createElement("div",Object(s.a)({key:t},a),e.map((function(e,t){return r.a.createElement("span",Object(s.a)({key:t},l({token:e,key:t})))})))})))),r.a.createElement("button",{ref:f,type:"button","aria-label":Object(O.b)({id:"theme.CodeBlock.copyButtonAriaLabel",message:"Copy code to clipboard",description:"The ARIA label for copy code blocks button"}),className:Object(d.a)(K.a.copyButton),onClick:B},i?r.a.createElement(O.a,{id:"theme.CodeBlock.copied",description:"The copied button label on code blocks"},"Copied"):r.a.createElement(O.a,{id:"theme.CodeBlock.copy",description:"The copy button label on code blocks"},"Copy"))))}))}a(87);var G=a(88),Q=a.n(G),Z=function(e){return function(t){var a,n=t.id,o=Object(u.a)(t,["id"]),c=Object(p.useThemeConfig)().navbar.hideOnScroll;return n?r.a.createElement(e,o,r.a.createElement("a",{"aria-hidden":"true",tabIndex:-1,className:Object(d.a)("anchor",(a={},a[Q.a.enhancedAnchor]=!c,a)),id:n}),o.children,r.a.createElement("a",{className:"hash-link",href:"#"+n,title:Object(O.b)({id:"theme.common.headingLinkTitle",message:"Direct link to heading",description:"Title for link to heading"})},"#")):r.a.createElement(e,o)}},ee={code:function(e){var t=e.children;return"string"==typeof t?t.includes("\n")?r.a.createElement(q,e):r.a.createElement("code",e):t},a:function(e){return r.a.createElement(y.a,e)},pre:function(e){var t=e.children;return r.a.createElement(q,Object(n.isValidElement)(t)?null==t?void 0:t.props:{children:t})},h1:Z("h1"),h2:Z("h2"),h3:Z("h3"),h4:Z("h4"),h5:Z("h5"),h6:Z("h6")},te=a(239),ae=a(23),ne=a(89),re=a.n(ne);function oe(e){var t,a,l,s,u,m=e.currentDocRoute,b=e.versionMetadata,h=e.children,f=Object(c.default)(),y=f.siteConfig,g=f.isClient,v=b.pluginId,k=b.permalinkToSidebar,E=b.docsSidebars,C=b.version,N=k[m.path],T=E[N],x=Object(n.useState)(!1),w=x[0],S=x[1],I=Object(n.useState)(!1),_=I[0],B=I[1],L=Object(n.useCallback)((function(){_&&B(!1),S(!w)}),[_]);return r.a.createElement(i.a,{key:g,wrapperClassName:"main-docs-wrapper",searchMetadatas:{version:C,tag:Object(p.docVersionSearchTag)(v,C)}},r.a.createElement("div",{className:re.a.docPage},T&&r.a.createElement("div",{className:Object(d.a)(re.a.docSidebarContainer,(t={},t[re.a.docSidebarContainerHidden]=w,t)),onTransitionEnd:function(e){e.currentTarget.classList.contains(re.a.docSidebarContainer)&&w&&B(!0)},role:"complementary"},r.a.createElement(P,{key:N,sidebar:T,path:m.path,sidebarCollapsible:null===(a=null===(l=y.themeConfig)||void 0===l?void 0:l.sidebarCollapsible)||void 0===a||a,onCollapse:L,isHidden:_}),_&&r.a.createElement("div",{className:re.a.collapsedDocSidebar,title:Object(O.b)({id:"theme.docs.sidebar.expandButtonTitle",message:"Expand sidebar",description:"The ARIA label and title attribute for expand button of doc sidebar"}),"aria-label":Object(O.b)({id:"theme.docs.sidebar.expandButtonAriaLabel",message:"Expand sidebar",description:"The ARIA label and title attribute for expand button of doc sidebar"}),tabIndex:0,role:"button",onKeyDown:L,onClick:L},r.a.createElement(j,{className:re.a.expandSidebarButtonIcon}))),r.a.createElement("main",{className:Object(d.a)(re.a.docMainContainer,(s={},s[re.a.docMainContainerEnhanced]=w,s))},r.a.createElement("div",{className:Object(d.a)("container padding-vert--lg",re.a.docItemWrapper,(u={},u[re.a.docItemWrapperEnhanced]=w,u))},r.a.createElement(o.a,{components:ee},h)))))}t.default=function(e){var t=e.route.routes,a=e.versionMetadata,n=e.location,o=t.find((function(e){return Object(ae.matchPath)(n.pathname,e)}));return o?r.a.createElement(oe,{currentDocRoute:o,versionMetadata:a},Object(l.a)(t)):r.a.createElement(te.default,e)}},228:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return b}));var n=a(0),r=a.n(n);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function c(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?c(Object(a),!0).forEach((function(t){o(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):c(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function i(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},o=Object.keys(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=r.a.createContext({}),u=function(e){var t=r.a.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},d=function(e){var t=u(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,c=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),d=u(a),m=n,b=d["".concat(c,".").concat(m)]||d[m]||p[m]||o;return a?r.a.createElement(b,l(l({ref:t},s),{},{components:a})):r.a.createElement(b,l({ref:t},s))}));function b(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,c=new Array(o);c[0]=m;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l.mdxType="string"==typeof e?e:n,c[1]=l;for(var s=2;s<o;s++)c[s]=a[s];return r.a.createElement.apply(null,c)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"},239:function(e,t,a){"use strict";a.r(t);var n=a(0),r=a.n(n),o=a(236),c=a(231);t.default=function(){return r.a.createElement(o.a,{title:"Page Not Found"},r.a.createElement("main",{className:"container margin-vert--xl"},r.a.createElement("div",{className:"row"},r.a.createElement("div",{className:"col col--6 col--offset-3"},r.a.createElement("h1",{className:"hero__title"},r.a.createElement(c.a,{id:"theme.NotFound.title",description:"The title of the 404 page"},"Page Not Found")),r.a.createElement("p",null,r.a.createElement(c.a,{id:"theme.NotFound.p1",description:"The first paragraph of the 404 page"},"We could not find what you were looking for.")),r.a.createElement("p",null,r.a.createElement(c.a,{id:"theme.NotFound.p2",description:"The 2nd paragraph of the 404 page"},"Please contact the owner of the site that linked you to the original URL and let them know their link is broken."))))))}},272:function(e,t){function a(e){let t,a=[];for(let n of e.split(",").map((e=>e.trim())))if(/^-?\d+$/.test(n))a.push(parseInt(n,10));else if(t=n.match(/^(-?\d+)(-|\.\.\.?|\u2025|\u2026|\u22EF)(-?\d+)$/)){let[e,n,r,o]=t;if(n&&o){n=parseInt(n),o=parseInt(o);const e=n<o?1:-1;"-"!==r&&".."!==r&&"\u2025"!==r||(o+=e);for(let t=n;t!==o;t+=e)a.push(t)}}return a}t.default=a,e.exports=a}}]);
\ No newline at end of file
diff --git a/assets/js/162.71673972.js b/assets/js/158.319f81c0.js
similarity index 80%
rename from assets/js/162.71673972.js
rename to assets/js/158.319f81c0.js
index 683d165c..14a445a8 100644
--- a/assets/js/162.71673972.js
+++ b/assets/js/158.319f81c0.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[162],{243:function(e,t,a){"use strict";a.r(t);var n=a(0),o=a.n(n),l=a(240),r=a(235);t.default=function(){return o.a.createElement(l.a,{title:"Page Not Found"},o.a.createElement("main",{className:"container margin-vert--xl"},o.a.createElement("div",{className:"row"},o.a.createElement("div",{className:"col col--6 col--offset-3"},o.a.createElement("h1",{className:"hero__title"},o.a.createElement(r.a,{id:"theme.NotFound.title",description:"The title of the 404 page"},"Page Not Found")),o.a.createElement("p",null,o.a.createElement(r.a,{id:"theme.NotFound.p1",description:"The first paragraph of the 404 page"},"We could not find what you were looking for.")),o.a.createElement("p",null,o.a.createElement(r.a,{id:"theme.NotFound.p2",description:"The 2nd paragraph of the 404 page"},"Please contact the owner of the site that linked you to the original URL and let them know their link is broken."))))))}}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[158],{239:function(e,t,a){"use strict";a.r(t);var n=a(0),o=a.n(n),l=a(236),r=a(231);t.default=function(){return o.a.createElement(l.a,{title:"Page Not Found"},o.a.createElement("main",{className:"container margin-vert--xl"},o.a.createElement("div",{className:"row"},o.a.createElement("div",{className:"col col--6 col--offset-3"},o.a.createElement("h1",{className:"hero__title"},o.a.createElement(r.a,{id:"theme.NotFound.title",description:"The title of the 404 page"},"Page Not Found")),o.a.createElement("p",null,o.a.createElement(r.a,{id:"theme.NotFound.p1",description:"The first paragraph of the 404 page"},"We could not find what you were looking for.")),o.a.createElement("p",null,o.a.createElement(r.a,{id:"theme.NotFound.p2",description:"The 2nd paragraph of the 404 page"},"Please contact the owner of the site that linked you to the original URL and let them know their link is broken."))))))}}}]);
\ No newline at end of file
diff --git a/assets/js/16fcefd4.2fe48b23.js b/assets/js/16fcefd4.9eb05a6f.js
similarity index 98%
rename from assets/js/16fcefd4.2fe48b23.js
rename to assets/js/16fcefd4.9eb05a6f.js
index 65a95ae7..2bc35915 100644
--- a/assets/js/16fcefd4.2fe48b23.js
+++ b/assets/js/16fcefd4.9eb05a6f.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[16],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},78:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/docs/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/tag.md",version:"current",sidebar_label:"Making Releases",sidebar:"Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/next/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[18],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},80:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/docs/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/tag.md",version:"current",sidebar_label:"Making Releases",sidebar:"Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/next/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/17896441.1b171940.js b/assets/js/17896441.104e0abb.js
similarity index 92%
rename from assets/js/17896441.1b171940.js
rename to assets/js/17896441.104e0abb.js
index d47dfb59..e109685a 100644
--- a/assets/js/17896441.1b171940.js
+++ b/assets/js/17896441.104e0abb.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[17],{231:function(e,t,a){"use strict";a.r(t);var n=a(0),l=a.n(n),r=a(237),i=a(235);var s=function(e){var t=e.metadata;return l.a.createElement("nav",{className:"pagination-nav","aria-label":Object(i.b)({id:"theme.docs.paginator.navAriaLabel",message:"Docs pages navigation",description:"The ARIA label for the docs pagination"})},l.a.createElement("div",{className:"pagination-nav__item"},t.previous&&l.a.createElement(r.a,{className:"pagination-nav__link",to:t.previous.permalink},l.a.createElement("div",{className:"pagination-nav__sublabel"},l.a.createElement(i.a,{id:"theme.docs.paginator.previous",description:"The label used to navigate to the previous doc"},"Previous")),l.a.createElement("div",{className:"pagination-nav__label"},"\xab ",t.previous.title))),l.a.createElement("div",{className:"pagination-nav__item pagination-nav__item--next"},t.next&&l.a.createElement(r.a,{className:"pagination-nav__link",to:t.next.permalink},l.a.createElement("div",{className:"pagination-nav__sublabel"},l.a.createElement(i.a,{id:"theme.docs.paginator.next",description:"The label used to navigate to the next doc"},"Next")),l.a.createElement("div",{className:"pagination-nav__label"},t.next.title," \xbb"))))},o=a(16),c=a(236),d=a(233);function m(e){var t=e.siteTitle,a=e.versionLabel;return l.a.createElement(i.a,{id:"theme.docs.versions.unreleasedVersionLabel",description:"The label used to tell the user that he's browsing an unreleased doc version",values:{siteTitle:t,versionLabel:l.a.createElement("strong",null,a)}},"This is unreleased documentation for {siteTitle} {versionLabel} version.")}function u(e){var t=e.siteTitle,a=e.versionLabel;return l.a.createElement(i.a,{id:"theme.docs.versions.unmaintainedVersionLabel",description:"The label used to tell the user that he's browsing an unmaintained doc version",values:{siteTitle:t,versionLabel:l.a.createElement("strong",null,a)}},"This is documentation for {siteTitle} {versionLabel}, which is no longer actively maintained.")}function v(e){var t=e.versionLabel,a=e.to,n=e.onClick;return l.a.createElement(i.a,{id:"theme.docs.versions.latestVersionSuggestionLabel",description:"The label userd to tell the user that he's browsing an unmaintained doc version",values:{versionLabel:t,latestVersionLink:l.a.createElement("strong",null,l.a.createElement(r.a,{to:a,onClick:n},l.a.createElement(i.a,{id:"theme.docs.versions.latestVersionLinkLabel",description:"The label used for the latest version suggestion link label"},"latest version")))}},"For up-to-date documentation, see the {latestVersionLink} ({versionLabel}).")}var p=function(){var e=Object(o.default)().siteConfig.title,t=Object(c.useActivePlugin)({failfast:!0}).pluginId,a=Object(d.useDocsPreferredVersion)(t).savePreferredVersionName,n=Object(c.useActiveVersion)(t),r=Object(c.useDocVersionSuggestions)(t),i=r.latestDocSuggestion,s=r.latestVersionSuggestion;if(!s)return l.a.createElement(l.a.Fragment,null);var p,b=null!=i?i:(p=s).docs.find((function(e){return e.id===p.mainDocId}));return l.a.createElement("div",{className:"alert alert--warning margin-bottom--md",role:"alert"},l.a.createElement("div",null,"current"===n.name?l.a.createElement(m,{siteTitle:e,versionLabel:n.label}):l.a.createElement(u,{siteTitle:e,versionLabel:n.label})),l.a.createElement("div",{className:"margin-top--md"},l.a.createElement(v,{versionLabel:s.label,to:b.path,onClick:function(){return a(s.name)}})))},b=a(257),g=a(79),E=a.n(g);function h(e){var t=e.lastUpdatedAt,a=e.formattedLastUpdatedAt;return l.a.createElement(i.a,{id:"theme.lastUpdated.atDate",description:"The words used to describe on which date a page has been last updated",values:{date:l.a.createElement("time",{dateTime:new Date(1e3*t).toISOString(),className:E.a.lastUpdatedDate},a)}}," on {date}")}function f(e){var t=e.lastUpdatedBy;return l.a.createElement(i.a,{id:"theme.lastUpdated.byUser",description:"The words used to describe by who the page has been last updated",values:{user:l.a.createElement("strong",null,t)}}," by {user}")}function N(e){var t=e.lastUpdatedAt,a=e.formattedLastUpdatedAt,n=e.lastUpdatedBy;return l.a.createElement("div",{className:"col text--right"},l.a.createElement("em",null,l.a.createElement("small",null,l.a.createElement(i.a,{id:"theme.lastUpdated.lastUpdatedAtBy",description:"The sentence used to display when a page has been last updated, and by who",values:{atDate:t&&a?l.a.createElement(h,{lastUpdatedAt:t,formattedLastUpdatedAt:a}):"",byUser:n?l.a.createElement(f,{lastUpdatedBy:n}):""}},"Last updated{atDate}{byUser}"),!1)))}var L=a(234);var _=function(e,t,a){var l=Object(n.useState)(void 0),r=l[0],i=l[1];Object(n.useEffect)((function(){function n(){var n=function(){var e=Array.from(document.getElementsByClassName("anchor")),t=e.find((function(e){return e.getBoundingClientRect().top>=a}));if(t){if(t.getBoundingClientRect().top>=a){var n=e[e.indexOf(t)-1];return null!=n?n:t}return t}return e[e.length-1]}();if(n)for(var l=0,s=!1,o=document.getElementsByClassName(e);l<o.length&&!s;){var c=o[l],d=c.href,m=decodeURIComponent(d.substring(d.indexOf("#")+1));n.id===m&&(r&&r.classList.remove(t),c.classList.add(t),i(c),s=!0),l+=1}}return document.addEventListener("scroll",n),document.addEventListener("resize",n),n(),function(){document.removeEventListener("scroll",n),document.removeEventListener("resize",n)}}))},U=a(80),w=a.n(U),T="table-of-contents__link";function y(e){var t=e.toc,a=e.isChild;return t.length?l.a.createElement("ul",{className:a?"":"table-of-contents table-of-contents__left-border"},t.map((function(e){return l.a.createElement("li",{key:e.id},l.a.createElement("a",{href:"#"+e.id,className:T,dangerouslySetInnerHTML:{__html:e.value}}),l.a.createElement(y,{isChild:!0,toc:e.children}))}))):null}var k=function(e){var t=e.toc;return _(T,"table-of-contents__link--active",100),l.a.createElement("div",{className:Object(L.a)(w.a.tableOfContents,"thin-scrollbar")},l.a.createElement(y,{toc:t}))},A=a(3),O=a(7),j=a(81),C=a.n(j),V=function(e){var t=e.className,a=Object(O.a)(e,["className"]);return l.a.createElement("svg",Object(A.a)({fill:"currentColor",height:"1.2em",width:"1.2em",preserveAspectRatio:"xMidYMid meet",role:"img",viewBox:"0 0 40 40",className:Object(L.a)(C.a.iconEdit,t),"aria-label":"Edit page"},a),l.a.createElement("g",null,l.a.createElement("path",{d:"m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"})))};function x(e){var t=e.editUrl;return l.a.createElement("a",{href:t,target:"_blank",rel:"noreferrer noopener"},l.a.createElement(V,null),l.a.createElement(i.a,{id:"theme.common.editThisPage",description:"The link label to edit the current page"},"Edit this page"))}var B=a(82),D=a.n(B);t.default=function(e){var t,a=e.content,n=a.metadata,r=a.frontMatter,i=r.image,o=r.keywords,d=r.hide_title,m=r.hide_table_of_contents,u=n.description,v=n.title,g=n.editUrl,E=n.lastUpdatedAt,h=n.formattedLastUpdatedAt,f=n.lastUpdatedBy,_=Object(c.useActivePlugin)({failfast:!0}).pluginId,U=Object(c.useVersions)(_),w=Object(c.useActiveVersion)(_),T=U.length>1;return l.a.createElement(l.a.Fragment,null,l.a.createElement(b.a,{title:v,description:u,keywords:o,image:i}),l.a.createElement("div",{className:"row"},l.a.createElement("div",{className:Object(L.a)("col",(t={},t[D.a.docItemCol]=!m,t))},l.a.createElement(p,null),l.a.createElement("div",{className:D.a.docItemContainer},l.a.createElement("article",null,T&&l.a.createElement("div",null,l.a.createElement("span",{className:"badge badge--secondary"},"Version: ",w.label)),!d&&l.a.createElement("header",null,l.a.createElement("h1",{className:D.a.docTitle},v)),l.a.createElement("div",{className:"markdown"},l.a.createElement(a,null))),(g||E||f)&&l.a.createElement("div",{className:"margin-vert--xl"},l.a.createElement("div",{className:"row"},l.a.createElement("div",{className:"col"},g&&l.a.createElement(x,{editUrl:g})),(E||f)&&l.a.createElement(N,{lastUpdatedAt:E,formattedLastUpdatedAt:h,lastUpdatedBy:f}))),l.a.createElement("div",{className:"margin-vert--lg"},l.a.createElement(s,{metadata:n})))),!m&&a.toc&&l.a.createElement("div",{className:"col col--3"},l.a.createElement(k,{toc:a.toc}))))}}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[19],{227:function(e,t,a){"use strict";a.r(t);var n=a(0),l=a.n(n),r=a(233),i=a(231);var s=function(e){var t=e.metadata;return l.a.createElement("nav",{className:"pagination-nav","aria-label":Object(i.b)({id:"theme.docs.paginator.navAriaLabel",message:"Docs pages navigation",description:"The ARIA label for the docs pagination"})},l.a.createElement("div",{className:"pagination-nav__item"},t.previous&&l.a.createElement(r.a,{className:"pagination-nav__link",to:t.previous.permalink},l.a.createElement("div",{className:"pagination-nav__sublabel"},l.a.createElement(i.a,{id:"theme.docs.paginator.previous",description:"The label used to navigate to the previous doc"},"Previous")),l.a.createElement("div",{className:"pagination-nav__label"},"\xab ",t.previous.title))),l.a.createElement("div",{className:"pagination-nav__item pagination-nav__item--next"},t.next&&l.a.createElement(r.a,{className:"pagination-nav__link",to:t.next.permalink},l.a.createElement("div",{className:"pagination-nav__sublabel"},l.a.createElement(i.a,{id:"theme.docs.paginator.next",description:"The label used to navigate to the next doc"},"Next")),l.a.createElement("div",{className:"pagination-nav__label"},t.next.title," \xbb"))))},o=a(16),c=a(232),d=a(229);function m(e){var t=e.siteTitle,a=e.versionLabel;return l.a.createElement(i.a,{id:"theme.docs.versions.unreleasedVersionLabel",description:"The label used to tell the user that he's browsing an unreleased doc version",values:{siteTitle:t,versionLabel:l.a.createElement("strong",null,a)}},"This is unreleased documentation for {siteTitle} {versionLabel} version.")}function u(e){var t=e.siteTitle,a=e.versionLabel;return l.a.createElement(i.a,{id:"theme.docs.versions.unmaintainedVersionLabel",description:"The label used to tell the user that he's browsing an unmaintained doc version",values:{siteTitle:t,versionLabel:l.a.createElement("strong",null,a)}},"This is documentation for {siteTitle} {versionLabel}, which is no longer actively maintained.")}function v(e){var t=e.versionLabel,a=e.to,n=e.onClick;return l.a.createElement(i.a,{id:"theme.docs.versions.latestVersionSuggestionLabel",description:"The label userd to tell the user that he's browsing an unmaintained doc version",values:{versionLabel:t,latestVersionLink:l.a.createElement("strong",null,l.a.createElement(r.a,{to:a,onClick:n},l.a.createElement(i.a,{id:"theme.docs.versions.latestVersionLinkLabel",description:"The label used for the latest version suggestion link label"},"latest version")))}},"For up-to-date documentation, see the {latestVersionLink} ({versionLabel}).")}var p=function(){var e=Object(o.default)().siteConfig.title,t=Object(c.useActivePlugin)({failfast:!0}).pluginId,a=Object(d.useDocsPreferredVersion)(t).savePreferredVersionName,n=Object(c.useActiveVersion)(t),r=Object(c.useDocVersionSuggestions)(t),i=r.latestDocSuggestion,s=r.latestVersionSuggestion;if(!s)return l.a.createElement(l.a.Fragment,null);var p,b=null!=i?i:(p=s).docs.find((function(e){return e.id===p.mainDocId}));return l.a.createElement("div",{className:"alert alert--warning margin-bottom--md",role:"alert"},l.a.createElement("div",null,"current"===n.name?l.a.createElement(m,{siteTitle:e,versionLabel:n.label}):l.a.createElement(u,{siteTitle:e,versionLabel:n.label})),l.a.createElement("div",{className:"margin-top--md"},l.a.createElement(v,{versionLabel:s.label,to:b.path,onClick:function(){return a(s.name)}})))},b=a(253),g=a(81),E=a.n(g);function h(e){var t=e.lastUpdatedAt,a=e.formattedLastUpdatedAt;return l.a.createElement(i.a,{id:"theme.lastUpdated.atDate",description:"The words used to describe on which date a page has been last updated",values:{date:l.a.createElement("time",{dateTime:new Date(1e3*t).toISOString(),className:E.a.lastUpdatedDate},a)}}," on {date}")}function f(e){var t=e.lastUpdatedBy;return l.a.createElement(i.a,{id:"theme.lastUpdated.byUser",description:"The words used to describe by who the page has been last updated",values:{user:l.a.createElement("strong",null,t)}}," by {user}")}function N(e){var t=e.lastUpdatedAt,a=e.formattedLastUpdatedAt,n=e.lastUpdatedBy;return l.a.createElement("div",{className:"col text--right"},l.a.createElement("em",null,l.a.createElement("small",null,l.a.createElement(i.a,{id:"theme.lastUpdated.lastUpdatedAtBy",description:"The sentence used to display when a page has been last updated, and by who",values:{atDate:t&&a?l.a.createElement(h,{lastUpdatedAt:t,formattedLastUpdatedAt:a}):"",byUser:n?l.a.createElement(f,{lastUpdatedBy:n}):""}},"Last updated{atDate}{byUser}"),!1)))}var L=a(230);var _=function(e,t,a){var l=Object(n.useState)(void 0),r=l[0],i=l[1];Object(n.useEffect)((function(){function n(){var n=function(){var e=Array.from(document.getElementsByClassName("anchor")),t=e.find((function(e){return e.getBoundingClientRect().top>=a}));if(t){if(t.getBoundingClientRect().top>=a){var n=e[e.indexOf(t)-1];return null!=n?n:t}return t}return e[e.length-1]}();if(n)for(var l=0,s=!1,o=document.getElementsByClassName(e);l<o.length&&!s;){var c=o[l],d=c.href,m=decodeURIComponent(d.substring(d.indexOf("#")+1));n.id===m&&(r&&r.classList.remove(t),c.classList.add(t),i(c),s=!0),l+=1}}return document.addEventListener("scroll",n),document.addEventListener("resize",n),n(),function(){document.removeEventListener("scroll",n),document.removeEventListener("resize",n)}}))},U=a(82),w=a.n(U),T="table-of-contents__link";function y(e){var t=e.toc,a=e.isChild;return t.length?l.a.createElement("ul",{className:a?"":"table-of-contents table-of-contents__left-border"},t.map((function(e){return l.a.createElement("li",{key:e.id},l.a.createElement("a",{href:"#"+e.id,className:T,dangerouslySetInnerHTML:{__html:e.value}}),l.a.createElement(y,{isChild:!0,toc:e.children}))}))):null}var k=function(e){var t=e.toc;return _(T,"table-of-contents__link--active",100),l.a.createElement("div",{className:Object(L.a)(w.a.tableOfContents,"thin-scrollbar")},l.a.createElement(y,{toc:t}))},A=a(3),O=a(7),j=a(83),C=a.n(j),V=function(e){var t=e.className,a=Object(O.a)(e,["className"]);return l.a.createElement("svg",Object(A.a)({fill:"currentColor",height:"1.2em",width:"1.2em",preserveAspectRatio:"xMidYMid meet",role:"img",viewBox:"0 0 40 40",className:Object(L.a)(C.a.iconEdit,t),"aria-label":"Edit page"},a),l.a.createElement("g",null,l.a.createElement("path",{d:"m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"})))};function x(e){var t=e.editUrl;return l.a.createElement("a",{href:t,target:"_blank",rel:"noreferrer noopener"},l.a.createElement(V,null),l.a.createElement(i.a,{id:"theme.common.editThisPage",description:"The link label to edit the current page"},"Edit this page"))}var B=a(84),D=a.n(B);t.default=function(e){var t,a=e.content,n=a.metadata,r=a.frontMatter,i=r.image,o=r.keywords,d=r.hide_title,m=r.hide_table_of_contents,u=n.description,v=n.title,g=n.editUrl,E=n.lastUpdatedAt,h=n.formattedLastUpdatedAt,f=n.lastUpdatedBy,_=Object(c.useActivePlugin)({failfast:!0}).pluginId,U=Object(c.useVersions)(_),w=Object(c.useActiveVersion)(_),T=U.length>1;return l.a.createElement(l.a.Fragment,null,l.a.createElement(b.a,{title:v,description:u,keywords:o,image:i}),l.a.createElement("div",{className:"row"},l.a.createElement("div",{className:Object(L.a)("col",(t={},t[D.a.docItemCol]=!m,t))},l.a.createElement(p,null),l.a.createElement("div",{className:D.a.docItemContainer},l.a.createElement("article",null,T&&l.a.createElement("div",null,l.a.createElement("span",{className:"badge badge--secondary"},"Version: ",w.label)),!d&&l.a.createElement("header",null,l.a.createElement("h1",{className:D.a.docTitle},v)),l.a.createElement("div",{className:"markdown"},l.a.createElement(a,null))),(g||E||f)&&l.a.createElement("div",{className:"margin-vert--xl"},l.a.createElement("div",{className:"row"},l.a.createElement("div",{className:"col"},g&&l.a.createElement(x,{editUrl:g})),(E||f)&&l.a.createElement(N,{lastUpdatedAt:E,formattedLastUpdatedAt:h,lastUpdatedBy:f}))),l.a.createElement("div",{className:"margin-vert--lg"},l.a.createElement(s,{metadata:n})))),!m&&a.toc&&l.a.createElement("div",{className:"col col--3"},l.a.createElement(k,{toc:a.toc}))))}}}]);
\ No newline at end of file
diff --git a/assets/js/1bbe794e.484f5992.js b/assets/js/1bbe794e.484f5992.js
deleted file mode 100644
index 2b3ce6a0..00000000
--- a/assets/js/1bbe794e.484f5992.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[18],{232:function(e,t,o){"use strict";o.d(t,"a",(function(){return d})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(o),b=n,f=d["".concat(a,".").concat(b)]||d[b]||p[b]||i;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"},83:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(232)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-2.0.0/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-2.0.0/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_troubleshooting.md",version:"2.0.0",sidebar_label:"Troubleshooting",sidebar:"version-2.0.0/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(i.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Errors:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(i.b)("inlineCode",{parentName:"li"},"docker ps")),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(i.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Disk space:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(i.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(i.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(i.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Memory:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/1cc43674.8da06d41.js b/assets/js/1cc43674.f9080c78.js
similarity index 96%
rename from assets/js/1cc43674.8da06d41.js
rename to assets/js/1cc43674.f9080c78.js
index 1f727bea..f2551500 100644
--- a/assets/js/1cc43674.8da06d41.js
+++ b/assets/js/1cc43674.f9080c78.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[19],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),a=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),s=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},f=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,i=e.parentName,l=p(e,["components","mdxType","originalType","parentName"]),d=s(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||o;return n?a.a.createElement(m,c(c({ref:t},l),{},{components:n})):a.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=f;var c={};for(var p in t)hasOwnProperty.call(t,p)&&(c[p]=t[p]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<o;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"},89:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return s}));var r=n(3),a=n(7),o=(n(0),n(232)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},c={unversionedId:"pipeline/nfdump",id:"pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/docs/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/nfdump.md",version:"current",sidebar_label:"Sflow/Netflow Data",sidebar:"Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],l={toc:p};function s(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(o.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(o.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(o.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(o.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(o.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(o.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(o.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}s.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[20],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),a=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),s=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},f=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,i=e.parentName,l=p(e,["components","mdxType","originalType","parentName"]),d=s(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||o;return n?a.a.createElement(m,c(c({ref:t},l),{},{components:n})):a.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=f;var c={};for(var p in t)hasOwnProperty.call(t,p)&&(c[p]=t[p]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<o;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"},90:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return s}));var r=n(3),a=n(7),o=(n(0),n(228)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},c={unversionedId:"pipeline/nfdump",id:"pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/docs/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/nfdump.md",version:"current",sidebar_label:"Sflow/Netflow Data",sidebar:"Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],l={toc:p};function s(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(o.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(o.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(o.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(o.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(o.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(o.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(o.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}s.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/1d038b8a.35918ded.js b/assets/js/1d038b8a.936f367a.js
similarity index 97%
rename from assets/js/1d038b8a.35918ded.js
rename to assets/js/1d038b8a.936f367a.js
index 041c8446..d0992caf 100644
--- a/assets/js/1d038b8a.35918ded.js
+++ b/assets/js/1d038b8a.936f367a.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[20],{232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?l(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):l(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),d=n,f=p["".concat(l,".").concat(d)]||p[d]||b[d]||a;return o?r.a.createElement(f,i(i({ref:t},c),{},{components:o})):r.a.createElement(f,i({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,l=new Array(a);l[0]=d;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c<a;c++)l[c]=o[c];return r.a.createElement.apply(null,l)}return r.a.createElement.apply(null,o)}d.displayName="MDXCreateElement"},90:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return l})),o.d(t,"metadata",(function(){return i})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(232)),l={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},i={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.9/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",version:"1.2.9",sidebar_label:"Troubleshooting",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(a.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(a.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Logstash or Importer errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure all containers are running."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the containers are running but nothing is happening and the disk is full, you probably need more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. ")),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(a.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[21],{228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?l(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):l(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),d=n,f=p["".concat(l,".").concat(d)]||p[d]||b[d]||a;return o?r.a.createElement(f,i(i({ref:t},c),{},{components:o})):r.a.createElement(f,i({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,l=new Array(a);l[0]=d;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c<a;c++)l[c]=o[c];return r.a.createElement.apply(null,l)}return r.a.createElement.apply(null,o)}d.displayName="MDXCreateElement"},91:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return l})),o.d(t,"metadata",(function(){return i})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(228)),l={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},i={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.9/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",version:"1.2.9",sidebar_label:"Troubleshooting",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(a.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(a.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Logstash or Importer errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure all containers are running."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the containers are running but nothing is happening and the disk is full, you probably need more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. ")),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(a.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/1f3a3353.117952bc.js b/assets/js/1f3a3353.117952bc.js
new file mode 100644
index 00000000..f8ef3da1
--- /dev/null
+++ b/assets/js/1f3a3353.117952bc.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[22],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},92:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/docs/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/docker.md",version:"current",sidebar_label:"Docker Dev Guide",sidebar:"Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/next/devel/docusaurus"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],l={toc:s};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,'You can use the "master" version or a tagged version.',Object(a.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(a.b)("h2",{id:"installing"},"Installing"),Object(a.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(a.b)("h2",{id:"importer"},"Importer"),Object(a.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(a.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(a.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/1f3a3353.c8d7370b.js b/assets/js/1f3a3353.c8d7370b.js
deleted file mode 100644
index 5bfb0451..00000000
--- a/assets/js/1f3a3353.c8d7370b.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[21],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(i,".").concat(u)]||p[u]||b[u]||a;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},91:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/docs/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/docker.md",version:"current",sidebar_label:"Docker Dev Guide",sidebar:"Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/next/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,'You can use the "master" version or a tagged version.',Object(a.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(a.b)("h2",{id:"installing"},"Installing"),Object(a.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(a.b)("h2",{id:"importer"},"Importer"),Object(a.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(a.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(a.b)("h2",{id:"build-images"},"Build Images"),Object(a.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(a.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/2.89c52fba.js b/assets/js/2.f7e8da3d.js
similarity index 90%
rename from assets/js/2.89c52fba.js
rename to assets/js/2.f7e8da3d.js
index d9912028..c3ec9197 100644
--- a/assets/js/2.89c52fba.js
+++ b/assets/js/2.f7e8da3d.js
@@ -1,2 +1,2 @@
-/*! For license information please see 2.89c52fba.js.LICENSE.txt */
-(window.webpackJsonp=window.webpackJsonp||[]).push([[2],{239:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(234),i=a(237),s=a(238),u=a(23),d=a(233);function m(e){var t=e.activeBasePath,a=e.activeBaseRegex,c=e.to,l=e.href,u=e.label,d=e.activeClassName,m=void 0===d?"navbar__link--active":d,f=e.prependBaseUrlToHref,h=Object(r.a)(e,["activeBasePath","activeBaseRegex","to","href","label","activeClassName","prependBaseUrlToHref"]),v=Object(s.a)(c),b=Object(s.a)(t),p=Object(s.a)(l,{forcePrependBaseUrl:!0});return o.a.createElement(i.a,Object(n.a)({},l?{href:f?p:l}:Object.assign({isNavLink:!0,activeClassName:m,to:v},t||a?{isActive:function(e,t){return a?new RegExp(a).test(t.pathname):t.pathname.startsWith(b)}}:null),h),u)}function f(e){var t,a=e.items,i=e.position,s=e.className,u=Object(r.a)(e,["items","position","className"]),d=Object(c.useRef)(null),f=Object(c.useRef)(null),h=Object(c.useState)(!1),v=h[0],b=h[1];Object(c.useEffect)((function(){var e=function(e){d.current&&!d.current.contains(e.target)&&b(!1)};return document.addEventListener("mousedown",e),document.addEventListener("touchstart",e),function(){document.removeEventListener("mousedown",e),document.removeEventListener("touchstart",e)}}),[d]);var p=function(e,t){return void 0===t&&(t=!1),Object(l.a)({"navbar__item navbar__link":!t,dropdown__link:t},e)};return a?o.a.createElement("div",{ref:d,className:Object(l.a)("navbar__item","dropdown","dropdown--hoverable",{"dropdown--left":"left"===i,"dropdown--right":"right"===i,"dropdown--show":v})},o.a.createElement(m,Object(n.a)({className:p(s)},u,{onClick:u.to?void 0:function(e){return e.preventDefault()},onKeyDown:function(e){"Enter"===e.key&&(e.preventDefault(),b(!v))}}),null!==(t=u.children)&&void 0!==t?t:u.label),o.a.createElement("ul",{ref:f,className:"dropdown__menu"},a.map((function(e,t){var c=e.className,l=Object(r.a)(e,["className"]);return o.a.createElement("li",{key:t},o.a.createElement(m,Object(n.a)({onKeyDown:function(e){if(t===a.length-1&&"Tab"===e.key){e.preventDefault(),b(!1);var n=d.current.nextElementSibling;n&&n.focus()}},activeClassName:"dropdown__link--active",className:p(c,!0)},l)))})))):o.a.createElement(m,Object(n.a)({className:p(s)},u))}function h(e){var t,a,i,s=e.items,f=e.className,h=(e.position,Object(r.a)(e,["items","className","position"])),v=Object(c.useRef)(null),b=Object(u.useLocation)().pathname,p=Object(c.useState)((function(){var e;return null===(e=!(null!=s&&s.some((function(e){return Object(d.isSamePath)(e.to,b)}))))||void 0===e||e})),g=p[0],k=p[1],E=function(e,t){return void 0===t&&(t=!1),Object(l.a)("menu__link",{"menu__link--sublist":t},e)};if(!s)return o.a.createElement("li",{className:"menu__list-item"},o.a.createElement(m,Object(n.a)({className:E(f)},h)));var O=null!==(t=v.current)&&void 0!==t&&t.scrollHeight?(null===(a=v.current)||void 0===a?void 0:a.scrollHeight)+"px":void 0;return o.a.createElement("li",{className:Object(l.a)("menu__list-item",{"menu__list-item--collapsed":g})},o.a.createElement(m,Object(n.a)({role:"button",className:E(f,!0)},h,{onClick:function(e){e.preventDefault(),k((function(e){return!e}))}}),null!==(i=h.children)&&void 0!==i?i:h.label),o.a.createElement("ul",{className:"menu__list",ref:v,style:{height:g?void 0:O}},s.map((function(e,t){var a=e.className,c=Object(r.a)(e,["className"]);return o.a.createElement("li",{className:"menu__list-item",key:t},o.a.createElement(m,Object(n.a)({activeClassName:"menu__link--active",className:E(a)},c,{onClick:h.onClick})))}))))}t.a=function(e){var t=e.mobile,a=void 0!==t&&t,n=Object(r.a)(e,["mobile"]),c=a?h:f;return o.a.createElement(c,n)}},240:function(e,t,a){"use strict";var n=a(0),r=a.n(n),c=a(234),o=a(235),l=a(23),i=a(56),s=a.n(i);function u(e){e.setAttribute("tabindex","-1"),e.focus(),e.removeAttribute("tabindex")}var d=function(){var e=Object(n.useRef)(null),t=Object(l.useLocation)();return Object(n.useEffect)((function(){t.hash||u(e.current)}),[t.pathname]),r.a.createElement("div",{ref:e},r.a.createElement("a",{href:"#main",className:s.a.skipToContent,onClick:function(e){e.preventDefault();var t=document.querySelector("main:first-of-type")||document.querySelector(".main-wrapper");t&&u(t)}},r.a.createElement(o.a,{id:"theme.common.skipToMainContent",description:"The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation"},"Skip to main content")))},m=a(233),f=a(247),h=a(57),v=a.n(h);var b=function(){var e,t=Object(f.a)(),a=t.isAnnouncementBarClosed,n=t.closeAnnouncementBar,l=Object(m.useThemeConfig)().announcementBar;if(!l)return null;var i=l.content,s=l.backgroundColor,u=l.textColor,d=l.isCloseable;return!i||d&&a?null:r.a.createElement("div",{className:v.a.announcementBar,style:{backgroundColor:s,color:u},role:"banner"},r.a.createElement("div",{className:Object(c.a)(v.a.announcementBarContent,(e={},e[v.a.announcementBarCloseable]=d,e)),dangerouslySetInnerHTML:{__html:i}}),d?r.a.createElement("button",{type:"button",className:v.a.announcementBarClose,onClick:n,"aria-label":Object(o.b)({id:"theme.AnnouncementBar.closeButtonAriaLabel",message:"Close",description:"The ARIA label for close button of announcement bar"})},r.a.createElement("span",{"aria-hidden":"true"},"\xd7")):null)},p=a(3),g=function(){return null},k=a(269),E=a.n(k),O=a(16),j=a(58),y=a.n(j),_=function(e){var t=e.icon,a=e.style;return r.a.createElement("span",{className:Object(c.a)(y.a.toggle,y.a.dark),style:a},t)},C=function(e){var t=e.icon,a=e.style;return r.a.createElement("span",{className:Object(c.a)(y.a.toggle,y.a.light),style:a},t)},w=function(e){var t=Object(m.useThemeConfig)().colorMode.switchConfig,a=t.darkIcon,n=t.darkIconStyle,c=t.lightIcon,o=t.lightIconStyle,l=Object(O.default)().isClient;return r.a.createElement(E.a,Object(p.a)({disabled:!l,icons:{checked:r.a.createElement(_,{icon:a,style:n}),unchecked:r.a.createElement(C,{icon:c,style:o})}},e))},N=a(241),T=a(251),S=function(e){var t=Object(l.useLocation)(),a=Object(n.useState)(e),r=a[0],c=a[1],o=Object(n.useRef)(!1),i=Object(n.useState)(0),s=i[0],u=i[1],d=Object(n.useState)(0),m=d[0],f=d[1],h=Object(n.useCallback)((function(e){null!==e&&f(e.getBoundingClientRect().height)}),[]);return Object(T.a)((function(t){var a=t.scrollY;if(e)if(a<m)c(!0);else{if(o.current)return o.current=!1,c(!1),void u(a);s&&0===a&&c(!0);var n=document.documentElement.scrollHeight-m,r=window.innerHeight;s&&a>=s?c(!1):a+r<n&&c(!0),u(a)}}),[s,m,o]),Object(n.useEffect)((function(){e&&s&&c(!0)}),[t.pathname]),Object(n.useEffect)((function(){e&&t.hash&&(o.current=!0)}),[t.hash]),{navbarRef:h,isNavbarVisible:r}},L=a(252),I=a(253),D=a(7),B=a(239),x=function(e){var t=e.width,a=void 0===t?20:t,n=e.height,c=void 0===n?20:n,o=Object(D.a)(e,["width","height"]);return r.a.createElement("svg",Object(p.a)({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",width:a,height:c},o),r.a.createElement("path",{fill:"currentColor",d:"M19.753 10.909c-.624-1.707-2.366-2.726-4.661-2.726-.09 0-.176.002-.262.006l-.016-2.063 3.525-.607c.115-.019.133-.119.109-.231-.023-.111-.167-.883-.188-.976-.027-.131-.102-.127-.207-.109-.104.018-3.25.461-3.25.461l-.013-2.078c-.001-.125-.069-.158-.194-.156l-1.025.016c-.105.002-.164.049-.162.148l.033 2.307s-3.061.527-3.144.543c-.084.014-.17.053-.151.143.019.09.19 1.094.208 1.172.018.08.072.129.188.107l2.924-.504.035 2.018c-1.077.281-1.801.824-2.256 1.303-.768.807-1.207 1.887-1.207 2.963 0 1.586.971 2.529 2.328 2.695 3.162.387 5.119-3.06 5.769-4.715 1.097 1.506.256 4.354-2.094 5.98-.043.029-.098.129-.033.207l.619.756c.08.096.206.059.256.023 2.51-1.73 3.661-4.515 2.869-6.683zm-7.386 3.188c-.966-.121-.944-.914-.944-1.453 0-.773.327-1.58.876-2.156a3.21 3.21 0 011.229-.799l.082 4.277a2.773 2.773 0 01-1.243.131zm2.427-.553l.046-4.109c.084-.004.166-.01.252-.01.773 0 1.494.145 1.885.361.391.217-1.023 2.713-2.183 3.758zm-8.95-7.668a.196.196 0 00-.196-.145h-1.95a.194.194 0 00-.194.144L.008 16.916c-.017.051-.011.076.062.076h1.733c.075 0 .099-.023.114-.072l1.008-3.318h3.496l1.008 3.318c.016.049.039.072.113.072h1.734c.072 0 .078-.025.062-.076-.014-.05-3.083-9.741-3.494-11.04zm-2.618 6.318l1.447-5.25 1.447 5.25H3.226z"}))};function A(e){var t=e.mobile,a=e.dropdownItemsBefore,n=e.dropdownItemsAfter,c=Object(D.a)(e,["mobile","dropdownItemsBefore","dropdownItemsAfter"]),o=Object(O.default)().i18n,l=o.currentLocale,i=o.locales,s=o.localeConfigs,u=Object(m.useAlternatePageUtils)();function d(e){return s[e].label}var f=i.map((function(e){var t="pathname://"+u.createUrl({locale:e,fullyQualified:!1});return{isNavLink:!0,label:d(e),to:t,target:"_self",autoAddBaseUrl:!1,className:e===l?"dropdown__link--active":""}})),h=[].concat(a,f,n),v=t?"Languages":d(l);return r.a.createElement(B.a,Object(p.a)({},c,{href:"#",mobile:t,label:r.a.createElement("span",null,r.a.createElement(x,{style:{verticalAlign:"text-bottom",marginRight:5}}),r.a.createElement("span",null,v)),items:h}))}var P=a(59),M=a.n(P);function V(e){return e.mobile?null:r.a.createElement("div",{className:M.a.searchWrapper},r.a.createElement(g,null))}var U={default:function(){return B.a},localeDropdown:function(){return A},search:function(){return V},docsVersion:function(){return a(273).default},docsVersionDropdown:function(){return a(274).default},doc:function(){return a(275).default}};function R(e){var t=e.type,a=Object(D.a)(e,["type"]),n=function(e){void 0===e&&(e="default");var t=U[e];if(!t)throw new Error("No NavbarItem component found for type="+e+".");return t()}(t);return r.a.createElement(n,a)}var F=a(254),H=a(256),X=a(61),z=a.n(X),G="right";var W=function(){var e,t=Object(m.useThemeConfig)(),a=t.navbar,o=a.items,l=a.hideOnScroll,i=a.style,s=t.colorMode.disableSwitch,u=Object(n.useState)(!1),d=u[0],f=u[1],h=Object(N.a)(),v=h.isDarkTheme,b=h.setLightTheme,k=h.setDarkTheme,E=S(l),O=E.navbarRef,j=E.isNavbarVisible;Object(L.a)(d);var y=Object(n.useCallback)((function(){f(!0)}),[f]),_=Object(n.useCallback)((function(){f(!1)}),[f]),C=Object(n.useCallback)((function(e){return e.target.checked?k():b()}),[b,k]),T=Object(I.a)();Object(n.useEffect)((function(){T===I.b.desktop&&f(!1)}),[T]);var D=o.some((function(e){return"search"===e.type})),B=function(e){return{leftItems:e.filter((function(e){var t;return"left"===(null!==(t=e.position)&&void 0!==t?t:G)})),rightItems:e.filter((function(e){var t;return"right"===(null!==(t=e.position)&&void 0!==t?t:G)}))}}(o),x=B.leftItems,A=B.rightItems;return r.a.createElement("nav",{ref:O,className:Object(c.a)("navbar","navbar--fixed-top",(e={"navbar--dark":"dark"===i,"navbar--primary":"primary"===i,"navbar-sidebar--show":d},e[z.a.navbarHideable]=l,e[z.a.navbarHidden]=l&&!j,e))},r.a.createElement("div",{className:"navbar__inner"},r.a.createElement("div",{className:"navbar__items"},null!=o&&0!==o.length&&r.a.createElement("button",{"aria-label":"Navigation bar toggle",className:"navbar__toggle",type:"button",tabIndex:0,onClick:y,onKeyDown:y},r.a.createElement(H.a,null)),r.a.createElement(F.a,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:Object(c.a)("navbar__title")}),x.map((function(e,t){return r.a.createElement(R,Object(p.a)({},e,{key:t}))}))),r.a.createElement("div",{className:"navbar__items navbar__items--right"},A.map((function(e,t){return r.a.createElement(R,Object(p.a)({},e,{key:t}))})),!s&&r.a.createElement(w,{className:z.a.displayOnlyInLargeViewport,"aria-label":"Dark mode toggle",checked:v,onChange:C}),!D&&r.a.createElement(g,null))),r.a.createElement("div",{role:"presentation",className:"navbar-sidebar__backdrop",onClick:_}),r.a.createElement("div",{className:"navbar-sidebar"},r.a.createElement("div",{className:"navbar-sidebar__brand"},r.a.createElement(F.a,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:"navbar__title",onClick:_}),!s&&d&&r.a.createElement(w,{"aria-label":"Dark mode toggle in sidebar",checked:v,onChange:C})),r.a.createElement("div",{className:"navbar-sidebar__items"},r.a.createElement("div",{className:"menu"},r.a.createElement("ul",{className:"menu__list"},o.map((function(e,t){return r.a.createElement(R,Object(p.a)({mobile:!0},e,{onClick:_,key:t}))})))))))},Y=a(237),K=a(238),Q=a(62),q=a.n(Q),J=a(255);function Z(e){var t=e.to,a=e.href,n=e.label,c=e.prependBaseUrlToHref,o=Object(D.a)(e,["to","href","label","prependBaseUrlToHref"]),l=Object(K.a)(t),i=Object(K.a)(a,{forcePrependBaseUrl:!0});return r.a.createElement(Y.a,Object(p.a)({className:"footer__link-item"},a?{href:c?i:a}:{to:l},o),n)}var $=function(e){var t=e.sources,a=e.alt;return r.a.createElement(J.a,{className:"footer__logo",alt:a,sources:t})};var ee=function(){var e=Object(m.useThemeConfig)().footer,t=e||{},a=t.copyright,n=t.links,o=void 0===n?[]:n,l=t.logo,i=void 0===l?{}:l,s={light:Object(K.a)(i.src),dark:Object(K.a)(i.srcDark||i.src)};return e?r.a.createElement("footer",{className:Object(c.a)("footer",{"footer--dark":"dark"===e.style})},r.a.createElement("div",{className:"container"},o&&o.length>0&&r.a.createElement("div",{className:"row footer__links"},o.map((function(e,t){return r.a.createElement("div",{key:t,className:"col footer__col"},null!=e.title?r.a.createElement("h4",{className:"footer__title"},e.title):null,null!=e.items&&Array.isArray(e.items)&&e.items.length>0?r.a.createElement("ul",{className:"footer__items"},e.items.map((function(e,t){return e.html?r.a.createElement("li",{key:t,className:"footer__item",dangerouslySetInnerHTML:{__html:e.html}}):r.a.createElement("li",{key:e.href||e.to,className:"footer__item"},r.a.createElement(Z,e))}))):null)}))),(i||a)&&r.a.createElement("div",{className:"footer__bottom text--center"},i&&(i.src||i.srcDark)&&r.a.createElement("div",{className:"margin-bottom--sm"},i.href?r.a.createElement(Y.a,{href:i.href,className:q.a.footerLogoLink},r.a.createElement($,{alt:i.alt,sources:s})):r.a.createElement($,{alt:i.alt,sources:s})),a?r.a.createElement("div",{className:"footer__copyright",dangerouslySetInnerHTML:{__html:a}}):null))):null},te=a(8),ae="light",ne="dark",re=function(e){return e===ne?ne:ae},ce=function(e){try{localStorage.setItem("theme",re(e))}catch(t){console.error(t)}},oe=function(){var e=Object(m.useThemeConfig)().colorMode,t=e.defaultMode,a=e.disableSwitch,r=e.respectPrefersColorScheme,c=Object(n.useState)(function(e){return te.a.canUseDOM?re(document.documentElement.getAttribute("data-theme")):re(e)}(t)),o=c[0],l=c[1],i=Object(n.useCallback)((function(){l(ae),ce(ae)}),[]),s=Object(n.useCallback)((function(){l(ne),ce(ne)}),[]);return Object(n.useEffect)((function(){document.documentElement.setAttribute("data-theme",re(o))}),[o]),Object(n.useEffect)((function(){if(!a)try{var e=localStorage.getItem("theme");null!==e&&l(re(e))}catch(t){console.error(t)}}),[l]),Object(n.useEffect)((function(){a&&!r||window.matchMedia("(prefers-color-scheme: dark)").addListener((function(e){var t=e.matches;l(t?ne:ae)}))}),[]),{isDarkTheme:o===ne,setLightTheme:i,setDarkTheme:s}},le=a(250);var ie=function(e){var t=oe(),a=t.isDarkTheme,n=t.setLightTheme,c=t.setDarkTheme;return r.a.createElement(le.a.Provider,{value:{isDarkTheme:a,setLightTheme:n,setDarkTheme:c}},e.children)},se="docusaurus.tab.",ue=function(){var e=Object(n.useState)({}),t=e[0],a=e[1],r=Object(n.useCallback)((function(e,t){try{localStorage.setItem("docusaurus.tab."+e,t)}catch(a){console.error(a)}}),[]);return Object(n.useEffect)((function(){try{for(var e={},t=0;t<localStorage.length;t+=1){var n=localStorage.key(t);if(n.startsWith(se))e[n.substring(se.length)]=localStorage.getItem(n)}a(e)}catch(r){console.error(r)}}),[]),{tabGroupChoices:t,setTabGroupChoices:function(e,t){a((function(a){var n;return Object.assign({},a,((n={})[e]=t,n))})),r(e,t)}}},de="docusaurus.announcement.dismiss",me="docusaurus.announcement.id",fe=function(){var e=Object(m.useThemeConfig)().announcementBar,t=Object(n.useState)(!0),a=t[0],r=t[1],c=Object(n.useCallback)((function(){localStorage.setItem(de,"true"),r(!0)}),[]);return Object(n.useEffect)((function(){if(e){var t=e.id,a=localStorage.getItem(me);"annoucement-bar"===a&&(a="announcement-bar");var n=t!==a;localStorage.setItem(me,t),n&&localStorage.setItem(de,"false"),(n||"false"===localStorage.getItem(de))&&r(!1)}}),[]),{isAnnouncementBarClosed:a,closeAnnouncementBar:c}},he=a(248);var ve=function(e){var t=ue(),a=t.tabGroupChoices,n=t.setTabGroupChoices,c=fe(),o=c.isAnnouncementBarClosed,l=c.closeAnnouncementBar;return r.a.createElement(he.a.Provider,{value:{tabGroupChoices:a,setTabGroupChoices:n,isAnnouncementBarClosed:o,closeAnnouncementBar:l}},e.children)};function be(e){var t=e.children;return r.a.createElement(ie,null,r.a.createElement(ve,null,r.a.createElement(m.DocsPreferredVersionContextProvider,null,t)))}var pe=a(25);function ge(e){var t=e.locale,a=e.version,n=e.tag;return r.a.createElement(pe.a,null,t&&r.a.createElement("meta",{name:"docusaurus_locale",content:""+t}),a&&r.a.createElement("meta",{name:"docusaurus_version",content:a}),n&&r.a.createElement("meta",{name:"docusaurus_tag",content:n}))}var ke=a(257);function Ee(){var e=Object(O.default)().i18n,t=e.defaultLocale,a=e.locales,n=Object(m.useAlternatePageUtils)();return r.a.createElement(pe.a,null,a.map((function(e){return r.a.createElement("link",{key:e,rel:"alternate",href:n.createUrl({locale:e,fullyQualified:!0}),hrefLang:e})})),r.a.createElement("link",{rel:"alternate",href:n.createUrl({locale:t,fullyQualified:!0}),hrefLang:"x-default"}))}function Oe(e){var t=e.permalink,a=Object(O.default)().siteConfig.url,n=function(){var e=Object(O.default)().siteConfig.url,t=Object(l.useLocation)().pathname;return e+Object(K.a)(t)}(),c=t?""+a+t:n;return r.a.createElement(pe.a,null,r.a.createElement("meta",{property:"og:url",content:c}),r.a.createElement("link",{rel:"canonical",href:c}))}function je(e){var t=Object(O.default)(),a=t.siteConfig,n=t.i18n,c=n.currentLocale,o=n.localeConfigs,l=a.favicon,i=a.themeConfig,s=i.image,u=i.metadatas,d=e.title,f=e.description,h=e.image,v=e.keywords,b=e.searchMetadatas,g=Object(K.a)(l),k=c,E=o[c].direction;return r.a.createElement(r.a.Fragment,null,r.a.createElement(pe.a,null,r.a.createElement("html",{lang:k,dir:E}),l&&r.a.createElement("link",{rel:"shortcut icon",href:g})),r.a.createElement(ke.a,{title:d,description:f,keywords:v,image:h||s}),r.a.createElement(Oe,null),r.a.createElement(Ee,null),r.a.createElement(ge,Object(p.a)({tag:m.DEFAULT_SEARCH_TAG,locale:c},b)),r.a.createElement(pe.a,null,u.map((function(e,t){return r.a.createElement("meta",Object(p.a)({key:"metadata_"+t},e))}))))}a(63);var ye=function(){Object(n.useEffect)((function(){var e="navigation-with-keyboard";function t(t){"keydown"===t.type&&"Tab"===t.key&&document.body.classList.add(e),"mousedown"===t.type&&document.body.classList.remove(e)}return document.addEventListener("keydown",t),document.addEventListener("mousedown",t),function(){document.body.classList.remove(e),document.removeEventListener("keydown",t),document.removeEventListener("mousedown",t)}}),[])};a(64);t.a=function(e){var t=e.children,a=e.noFooter,n=e.wrapperClassName;return ye(),r.a.createElement(be,null,r.a.createElement(je,e),r.a.createElement(d,null),r.a.createElement(b,null),r.a.createElement(W,null),r.a.createElement("div",{className:Object(c.a)("main-wrapper",n)},t),!a&&r.a.createElement(ee,null))}},241:function(e,t,a){"use strict";var n=a(0),r=a(250);t.a=function(){var e=Object(n.useContext)(r.a);if(null==e)throw new Error("`useThemeContext` is used outside of `Layout` Component. See https://v2.docusaurus.io/docs/api/themes/configuration#usethemecontext.");return e}},247:function(e,t,a){"use strict";var n=a(0),r=a(248);t.a=function(){var e=Object(n.useContext)(r.a);if(null==e)throw new Error("`useUserPreferencesContext` is used outside of `Layout` Component.");return e}},248:function(e,t,a){"use strict";var n=a(0),r=Object(n.createContext)(void 0);t.a=r},249:function(e,t,a){var n;!function(){"use strict";var a={}.hasOwnProperty;function r(){for(var e=[],t=0;t<arguments.length;t++){var n=arguments[t];if(n){var c=typeof n;if("string"===c||"number"===c)e.push(n);else if(Array.isArray(n)&&n.length){var o=r.apply(null,n);o&&e.push(o)}else if("object"===c)for(var l in n)a.call(n,l)&&n[l]&&e.push(l)}}return e.join(" ")}e.exports?(r.default=r,e.exports=r):void 0===(n=function(){return r}.apply(t,[]))||(e.exports=n)}()},250:function(e,t,a){"use strict";var n=a(0),r=a.n(n).a.createContext(void 0);t.a=r},251:function(e,t,a){"use strict";var n=a(0),r=a(8),c=function(){return{scrollX:r.a.canUseDOM?window.pageXOffset:0,scrollY:r.a.canUseDOM?window.pageYOffset:0}};t.a=function(e,t){void 0===t&&(t=[]);var a=Object(n.useState)(c()),r=a[0],o=a[1],l=function(){var t=c();o(t),e&&e(t)};return Object(n.useEffect)((function(){var e={passive:!0};return window.addEventListener("scroll",l,e),function(){return window.removeEventListener("scroll",l,e)}}),t),r}},252:function(e,t,a){"use strict";var n=a(0);t.a=function(e){void 0===e&&(e=!0),Object(n.useEffect)((function(){return document.body.style.overflow=e?"hidden":"visible",function(){document.body.style.overflow="visible"}}),[e])}},253:function(e,t,a){"use strict";a.d(t,"b",(function(){return c}));var n=a(0),r=a(8),c={desktop:"desktop",mobile:"mobile"};t.a=function(){var e=r.a.canUseDOM;function t(){if(e)return window.innerWidth>996?c.desktop:c.mobile}var a=Object(n.useState)(t),o=a[0],l=a[1];return Object(n.useEffect)((function(){if(e)return window.addEventListener("resize",a),function(){return window.removeEventListener("resize",a)};function a(){l(t())}}),[]),o}},254:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(237),i=a(255),s=a(238),u=a(16),d=a(233);t.a=function(e){var t=Object(u.default)().isClient,a=Object(d.useThemeConfig)().navbar,c=a.title,m=a.logo,f=void 0===m?{src:""}:m,h=e.imageClassName,v=e.titleClassName,b=Object(r.a)(e,["imageClassName","titleClassName"]),p=Object(s.a)(f.href||"/"),g={light:Object(s.a)(f.src),dark:Object(s.a)(f.srcDark||f.src)};return o.a.createElement(l.a,Object(n.a)({to:p},b,f.target&&{target:f.target}),f.src&&o.a.createElement(i.a,{key:t,className:h,sources:g,alt:f.alt||c||"Logo"}),null!=c&&o.a.createElement("strong",{className:v},c))}},255:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(234),i=a(16),s=a(241),u=a(60),d=a.n(u);t.a=function(e){var t=Object(i.default)().isClient,a=Object(s.a)().isDarkTheme,c=e.sources,u=e.className,m=e.alt,f=void 0===m?"":m,h=Object(r.a)(e,["sources","className","alt"]),v=t?a?["dark"]:["light"]:["light","dark"];return o.a.createElement(o.a.Fragment,null,v.map((function(e){return o.a.createElement("img",Object(n.a)({key:e,src:c[e],alt:f,className:Object(l.a)(d.a.themedImage,d.a["themedImage--"+e],u)},h))})))}},256:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c);t.a=function(e){var t=e.width,a=void 0===t?30:t,c=e.height,l=void 0===c?30:c,i=e.className,s=Object(r.a)(e,["width","height","className"]);return o.a.createElement("svg",Object(n.a)({"aria-label":"Menu",className:i,width:a,height:l,viewBox:"0 0 30 30",role:"img",focusable:"false"},s),o.a.createElement("title",null,"Menu"),o.a.createElement("path",{stroke:"currentColor",strokeLinecap:"round",strokeMiterlimit:"10",strokeWidth:"2",d:"M4 7h22M4 15h22M4 23h22"}))}},269:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var a=arguments[t];for(var n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n])}return e},r=function(){function e(e,t){for(var a=0;a<t.length;a++){var n=t[a];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,a,n){return a&&e(t.prototype,a),n&&e(t,n),t}}(),c=a(0),o=m(c),l=m(a(249)),i=m(a(1)),s=m(a(270)),u=m(a(271)),d=a(272);function m(e){return e&&e.__esModule?e:{default:e}}var f=function(e){function t(e){!function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}(this,t);var a=function(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e));return a.handleClick=a.handleClick.bind(a),a.handleTouchStart=a.handleTouchStart.bind(a),a.handleTouchMove=a.handleTouchMove.bind(a),a.handleTouchEnd=a.handleTouchEnd.bind(a),a.handleFocus=a.handleFocus.bind(a),a.handleBlur=a.handleBlur.bind(a),a.previouslyChecked=!(!e.checked&&!e.defaultChecked),a.state={checked:!(!e.checked&&!e.defaultChecked),hasFocus:!1},a}return function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}(t,e),r(t,[{key:"componentDidUpdate",value:function(e){e.checked!==this.props.checked&&this.setState({checked:!!this.props.checked})}},{key:"handleClick",value:function(e){if(!this.props.disabled){var t=this.input;if(e.target!==t&&!this.moved)return this.previouslyChecked=t.checked,e.preventDefault(),t.focus(),void t.click();var a=this.props.hasOwnProperty("checked")?this.props.checked:t.checked;this.setState({checked:a})}}},{key:"handleTouchStart",value:function(e){this.props.disabled||(this.startX=(0,d.pointerCoord)(e).x,this.activated=!0)}},{key:"handleTouchMove",value:function(e){if(this.activated&&(this.moved=!0,this.startX)){var t=(0,d.pointerCoord)(e).x;this.state.checked&&t+15<this.startX?(this.setState({checked:!1}),this.startX=t,this.activated=!0):t-15>this.startX&&(this.setState({checked:!0}),this.startX=t,this.activated=t<this.startX+5)}}},{key:"handleTouchEnd",value:function(e){if(this.moved){var t=this.input;if(e.preventDefault(),this.startX){var a=(0,d.pointerCoord)(e).x;!0===this.previouslyChecked&&this.startX+4>a?this.previouslyChecked!==this.state.checked&&(this.setState({checked:!1}),this.previouslyChecked=this.state.checked,t.click()):this.startX-4<a&&this.previouslyChecked!==this.state.checked&&(this.setState({checked:!0}),this.previouslyChecked=this.state.checked,t.click()),this.activated=!1,this.startX=null,this.moved=!1}}}},{key:"handleFocus",value:function(e){var t=this.props.onFocus;t&&t(e),this.setState({hasFocus:!0})}},{key:"handleBlur",value:function(e){var t=this.props.onBlur;t&&t(e),this.setState({hasFocus:!1})}},{key:"getIcon",value:function(e){var a=this.props.icons;return a?void 0===a[e]?t.defaultProps.icons[e]:a[e]:null}},{key:"render",value:function(){var e=this,t=this.props,a=t.className,r=(t.icons,function(e,t){var a={};for(var n in e)t.indexOf(n)>=0||Object.prototype.hasOwnProperty.call(e,n)&&(a[n]=e[n]);return a}(t,["className","icons"])),c=(0,l.default)("react-toggle",{"react-toggle--checked":this.state.checked,"react-toggle--focus":this.state.hasFocus,"react-toggle--disabled":this.props.disabled},a);return o.default.createElement("div",{className:c,onClick:this.handleClick,onTouchStart:this.handleTouchStart,onTouchMove:this.handleTouchMove,onTouchEnd:this.handleTouchEnd},o.default.createElement("div",{className:"react-toggle-track"},o.default.createElement("div",{className:"react-toggle-track-check"},this.getIcon("checked")),o.default.createElement("div",{className:"react-toggle-track-x"},this.getIcon("unchecked"))),o.default.createElement("div",{className:"react-toggle-thumb"}),o.default.createElement("input",n({},r,{ref:function(t){e.input=t},onFocus:this.handleFocus,onBlur:this.handleBlur,className:"react-toggle-screenreader-only",type:"checkbox"})))}}]),t}(c.PureComponent);t.default=f,f.displayName="Toggle",f.defaultProps={icons:{checked:o.default.createElement(s.default,null),unchecked:o.default.createElement(u.default,null)}},f.propTypes={checked:i.default.bool,disabled:i.default.bool,defaultChecked:i.default.bool,onChange:i.default.func,onFocus:i.default.func,onBlur:i.default.func,className:i.default.string,name:i.default.string,value:i.default.string,id:i.default.string,"aria-labelledby":i.default.string,"aria-label":i.default.string,icons:i.default.oneOfType([i.default.bool,i.default.shape({checked:i.default.node,unchecked:i.default.node})])}},270:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n,r=a(0),c=(n=r)&&n.__esModule?n:{default:n};t.default=function(){return c.default.createElement("svg",{width:"14",height:"11",viewBox:"0 0 14 11"},c.default.createElement("path",{d:"M11.264 0L5.26 6.004 2.103 2.847 0 4.95l5.26 5.26 8.108-8.107L11.264 0",fill:"#fff",fillRule:"evenodd"}))}},271:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n,r=a(0),c=(n=r)&&n.__esModule?n:{default:n};t.default=function(){return c.default.createElement("svg",{width:"10",height:"10",viewBox:"0 0 10 10"},c.default.createElement("path",{d:"M9.9 2.12L7.78 0 4.95 2.828 2.12 0 0 2.12l2.83 2.83L0 7.776 2.123 9.9 4.95 7.07 7.78 9.9 9.9 7.776 7.072 4.95 9.9 2.12",fill:"#fff",fillRule:"evenodd"}))}},272:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.pointerCoord=function(e){if(e){var t=e.changedTouches;if(t&&t.length>0){var a=t[0];return{x:a.clientX,y:a.clientY}}var n=e.pageX;if(void 0!==n)return{x:n,y:e.pageY}}return{x:0,y:0}}},273:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return u}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(239),i=a(236),s=a(233);function u(e){var t,a=e.label,c=e.to,u=e.docsPluginId,d=Object(r.a)(e,["label","to","docsPluginId"]),m=Object(i.useActiveVersion)(u),f=Object(s.useDocsPreferredVersion)(u).preferredVersion,h=Object(i.useLatestVersion)(u),v=null!==(t=null!=m?m:f)&&void 0!==t?t:h,b=null!=a?a:v.label,p=null!=c?c:function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))}(v).path;return o.a.createElement(l.a,Object(n.a)({},d,{label:b,to:p}))}},274:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return d}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(239),i=a(236),s=a(233),u=function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))};function d(e){var t,a,c=e.mobile,d=e.docsPluginId,m=e.dropdownActiveClassDisabled,f=e.dropdownItemsBefore,h=e.dropdownItemsAfter,v=Object(r.a)(e,["mobile","docsPluginId","dropdownActiveClassDisabled","dropdownItemsBefore","dropdownItemsAfter"]),b=Object(i.useActiveDocContext)(d),p=Object(i.useVersions)(d),g=Object(i.useLatestVersion)(d),k=Object(s.useDocsPreferredVersion)(d),E=k.preferredVersion,O=k.savePreferredVersionName;var j=null!==(t=null!==(a=b.activeVersion)&&void 0!==a?a:E)&&void 0!==t?t:g,y=c?"Versions":j.label,_=c?void 0:u(j).path;return o.a.createElement(l.a,Object(n.a)({},v,{mobile:c,label:y,to:_,items:function(){var e=p.map((function(e){var t=(null==b?void 0:b.alternateDocVersions[e.name])||u(e);return{isNavLink:!0,label:e.label,to:t.path,isActive:function(){return e===(null==b?void 0:b.activeVersion)},onClick:function(){O(e.name)}}})),t=[].concat(f,e,h);if(!(t.length<=1))return t}(),isActive:m?function(){return!1}:void 0}))}},275:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return d}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(239),i=a(236),s=a(234),u=a(233);function d(e){var t,a,c=e.docId,d=e.activeSidebarClassName,m=e.label,f=e.docsPluginId,h=Object(r.a)(e,["docId","activeSidebarClassName","label","docsPluginId"]),v=Object(i.useActiveDocContext)(f),b=v.activeVersion,p=v.activeDoc,g=Object(u.useDocsPreferredVersion)(f).preferredVersion,k=Object(i.useLatestVersion)(f),E=null!==(t=null!=b?b:g)&&void 0!==t?t:k,O=E.docs.find((function(e){return e.id===c}));if(!O)throw new Error("DocNavbarItem: couldn't find any doc with id="+c+" in version "+E.name+".\nAvailable docIds=\n- "+E.docs.join("\n- "));return o.a.createElement(l.a,Object(n.a)({exact:!0},h,{className:Object(s.a)(h.className,(a={},a[d]=p&&p.sidebar===O.sidebar,a)),label:null!=m?m:O.id,to:O.path}))}}}]);
\ No newline at end of file
+/*! For license information please see 2.f7e8da3d.js.LICENSE.txt */
+(window.webpackJsonp=window.webpackJsonp||[]).push([[2],{235:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(230),i=a(233),s=a(234),u=a(23),d=a(229);function m(e){var t=e.activeBasePath,a=e.activeBaseRegex,c=e.to,l=e.href,u=e.label,d=e.activeClassName,m=void 0===d?"navbar__link--active":d,f=e.prependBaseUrlToHref,h=Object(r.a)(e,["activeBasePath","activeBaseRegex","to","href","label","activeClassName","prependBaseUrlToHref"]),v=Object(s.a)(c),b=Object(s.a)(t),p=Object(s.a)(l,{forcePrependBaseUrl:!0});return o.a.createElement(i.a,Object(n.a)({},l?{href:f?p:l}:Object.assign({isNavLink:!0,activeClassName:m,to:v},t||a?{isActive:function(e,t){return a?new RegExp(a).test(t.pathname):t.pathname.startsWith(b)}}:null),h),u)}function f(e){var t,a=e.items,i=e.position,s=e.className,u=Object(r.a)(e,["items","position","className"]),d=Object(c.useRef)(null),f=Object(c.useRef)(null),h=Object(c.useState)(!1),v=h[0],b=h[1];Object(c.useEffect)((function(){var e=function(e){d.current&&!d.current.contains(e.target)&&b(!1)};return document.addEventListener("mousedown",e),document.addEventListener("touchstart",e),function(){document.removeEventListener("mousedown",e),document.removeEventListener("touchstart",e)}}),[d]);var p=function(e,t){return void 0===t&&(t=!1),Object(l.a)({"navbar__item navbar__link":!t,dropdown__link:t},e)};return a?o.a.createElement("div",{ref:d,className:Object(l.a)("navbar__item","dropdown","dropdown--hoverable",{"dropdown--left":"left"===i,"dropdown--right":"right"===i,"dropdown--show":v})},o.a.createElement(m,Object(n.a)({className:p(s)},u,{onClick:u.to?void 0:function(e){return e.preventDefault()},onKeyDown:function(e){"Enter"===e.key&&(e.preventDefault(),b(!v))}}),null!==(t=u.children)&&void 0!==t?t:u.label),o.a.createElement("ul",{ref:f,className:"dropdown__menu"},a.map((function(e,t){var c=e.className,l=Object(r.a)(e,["className"]);return o.a.createElement("li",{key:t},o.a.createElement(m,Object(n.a)({onKeyDown:function(e){if(t===a.length-1&&"Tab"===e.key){e.preventDefault(),b(!1);var n=d.current.nextElementSibling;n&&n.focus()}},activeClassName:"dropdown__link--active",className:p(c,!0)},l)))})))):o.a.createElement(m,Object(n.a)({className:p(s)},u))}function h(e){var t,a,i,s=e.items,f=e.className,h=(e.position,Object(r.a)(e,["items","className","position"])),v=Object(c.useRef)(null),b=Object(u.useLocation)().pathname,p=Object(c.useState)((function(){var e;return null===(e=!(null!=s&&s.some((function(e){return Object(d.isSamePath)(e.to,b)}))))||void 0===e||e})),g=p[0],k=p[1],E=function(e,t){return void 0===t&&(t=!1),Object(l.a)("menu__link",{"menu__link--sublist":t},e)};if(!s)return o.a.createElement("li",{className:"menu__list-item"},o.a.createElement(m,Object(n.a)({className:E(f)},h)));var O=null!==(t=v.current)&&void 0!==t&&t.scrollHeight?(null===(a=v.current)||void 0===a?void 0:a.scrollHeight)+"px":void 0;return o.a.createElement("li",{className:Object(l.a)("menu__list-item",{"menu__list-item--collapsed":g})},o.a.createElement(m,Object(n.a)({role:"button",className:E(f,!0)},h,{onClick:function(e){e.preventDefault(),k((function(e){return!e}))}}),null!==(i=h.children)&&void 0!==i?i:h.label),o.a.createElement("ul",{className:"menu__list",ref:v,style:{height:g?void 0:O}},s.map((function(e,t){var a=e.className,c=Object(r.a)(e,["className"]);return o.a.createElement("li",{className:"menu__list-item",key:t},o.a.createElement(m,Object(n.a)({activeClassName:"menu__link--active",className:E(a)},c,{onClick:h.onClick})))}))))}t.a=function(e){var t=e.mobile,a=void 0!==t&&t,n=Object(r.a)(e,["mobile"]),c=a?h:f;return o.a.createElement(c,n)}},236:function(e,t,a){"use strict";var n=a(0),r=a.n(n),c=a(230),o=a(231),l=a(23),i=a(56),s=a.n(i);function u(e){e.setAttribute("tabindex","-1"),e.focus(),e.removeAttribute("tabindex")}var d=function(){var e=Object(n.useRef)(null),t=Object(l.useLocation)();return Object(n.useEffect)((function(){t.hash||u(e.current)}),[t.pathname]),r.a.createElement("div",{ref:e},r.a.createElement("a",{href:"#main",className:s.a.skipToContent,onClick:function(e){e.preventDefault();var t=document.querySelector("main:first-of-type")||document.querySelector(".main-wrapper");t&&u(t)}},r.a.createElement(o.a,{id:"theme.common.skipToMainContent",description:"The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation"},"Skip to main content")))},m=a(229),f=a(243),h=a(57),v=a.n(h);var b=function(){var e,t=Object(f.a)(),a=t.isAnnouncementBarClosed,n=t.closeAnnouncementBar,l=Object(m.useThemeConfig)().announcementBar;if(!l)return null;var i=l.content,s=l.backgroundColor,u=l.textColor,d=l.isCloseable;return!i||d&&a?null:r.a.createElement("div",{className:v.a.announcementBar,style:{backgroundColor:s,color:u},role:"banner"},r.a.createElement("div",{className:Object(c.a)(v.a.announcementBarContent,(e={},e[v.a.announcementBarCloseable]=d,e)),dangerouslySetInnerHTML:{__html:i}}),d?r.a.createElement("button",{type:"button",className:v.a.announcementBarClose,onClick:n,"aria-label":Object(o.b)({id:"theme.AnnouncementBar.closeButtonAriaLabel",message:"Close",description:"The ARIA label for close button of announcement bar"})},r.a.createElement("span",{"aria-hidden":"true"},"\xd7")):null)},p=a(3),g=function(){return null},k=a(265),E=a.n(k),O=a(16),j=a(58),y=a.n(j),_=function(e){var t=e.icon,a=e.style;return r.a.createElement("span",{className:Object(c.a)(y.a.toggle,y.a.dark),style:a},t)},C=function(e){var t=e.icon,a=e.style;return r.a.createElement("span",{className:Object(c.a)(y.a.toggle,y.a.light),style:a},t)},w=function(e){var t=Object(m.useThemeConfig)().colorMode.switchConfig,a=t.darkIcon,n=t.darkIconStyle,c=t.lightIcon,o=t.lightIconStyle,l=Object(O.default)().isClient;return r.a.createElement(E.a,Object(p.a)({disabled:!l,icons:{checked:r.a.createElement(_,{icon:a,style:n}),unchecked:r.a.createElement(C,{icon:c,style:o})}},e))},N=a(237),T=a(247),S=function(e){var t=Object(l.useLocation)(),a=Object(n.useState)(e),r=a[0],c=a[1],o=Object(n.useRef)(!1),i=Object(n.useState)(0),s=i[0],u=i[1],d=Object(n.useState)(0),m=d[0],f=d[1],h=Object(n.useCallback)((function(e){null!==e&&f(e.getBoundingClientRect().height)}),[]);return Object(T.a)((function(t){var a=t.scrollY;if(e)if(a<m)c(!0);else{if(o.current)return o.current=!1,c(!1),void u(a);s&&0===a&&c(!0);var n=document.documentElement.scrollHeight-m,r=window.innerHeight;s&&a>=s?c(!1):a+r<n&&c(!0),u(a)}}),[s,m,o]),Object(n.useEffect)((function(){e&&s&&c(!0)}),[t.pathname]),Object(n.useEffect)((function(){e&&t.hash&&(o.current=!0)}),[t.hash]),{navbarRef:h,isNavbarVisible:r}},L=a(248),I=a(249),D=a(7),B=a(235),x=function(e){var t=e.width,a=void 0===t?20:t,n=e.height,c=void 0===n?20:n,o=Object(D.a)(e,["width","height"]);return r.a.createElement("svg",Object(p.a)({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",width:a,height:c},o),r.a.createElement("path",{fill:"currentColor",d:"M19.753 10.909c-.624-1.707-2.366-2.726-4.661-2.726-.09 0-.176.002-.262.006l-.016-2.063 3.525-.607c.115-.019.133-.119.109-.231-.023-.111-.167-.883-.188-.976-.027-.131-.102-.127-.207-.109-.104.018-3.25.461-3.25.461l-.013-2.078c-.001-.125-.069-.158-.194-.156l-1.025.016c-.105.002-.164.049-.162.148l.033 2.307s-3.061.527-3.144.543c-.084.014-.17.053-.151.143.019.09.19 1.094.208 1.172.018.08.072.129.188.107l2.924-.504.035 2.018c-1.077.281-1.801.824-2.256 1.303-.768.807-1.207 1.887-1.207 2.963 0 1.586.971 2.529 2.328 2.695 3.162.387 5.119-3.06 5.769-4.715 1.097 1.506.256 4.354-2.094 5.98-.043.029-.098.129-.033.207l.619.756c.08.096.206.059.256.023 2.51-1.73 3.661-4.515 2.869-6.683zm-7.386 3.188c-.966-.121-.944-.914-.944-1.453 0-.773.327-1.58.876-2.156a3.21 3.21 0 011.229-.799l.082 4.277a2.773 2.773 0 01-1.243.131zm2.427-.553l.046-4.109c.084-.004.166-.01.252-.01.773 0 1.494.145 1.885.361.391.217-1.023 2.713-2.183 3.758zm-8.95-7.668a.196.196 0 00-.196-.145h-1.95a.194.194 0 00-.194.144L.008 16.916c-.017.051-.011.076.062.076h1.733c.075 0 .099-.023.114-.072l1.008-3.318h3.496l1.008 3.318c.016.049.039.072.113.072h1.734c.072 0 .078-.025.062-.076-.014-.05-3.083-9.741-3.494-11.04zm-2.618 6.318l1.447-5.25 1.447 5.25H3.226z"}))};function A(e){var t=e.mobile,a=e.dropdownItemsBefore,n=e.dropdownItemsAfter,c=Object(D.a)(e,["mobile","dropdownItemsBefore","dropdownItemsAfter"]),o=Object(O.default)().i18n,l=o.currentLocale,i=o.locales,s=o.localeConfigs,u=Object(m.useAlternatePageUtils)();function d(e){return s[e].label}var f=i.map((function(e){var t="pathname://"+u.createUrl({locale:e,fullyQualified:!1});return{isNavLink:!0,label:d(e),to:t,target:"_self",autoAddBaseUrl:!1,className:e===l?"dropdown__link--active":""}})),h=[].concat(a,f,n),v=t?"Languages":d(l);return r.a.createElement(B.a,Object(p.a)({},c,{href:"#",mobile:t,label:r.a.createElement("span",null,r.a.createElement(x,{style:{verticalAlign:"text-bottom",marginRight:5}}),r.a.createElement("span",null,v)),items:h}))}var P=a(59),M=a.n(P);function V(e){return e.mobile?null:r.a.createElement("div",{className:M.a.searchWrapper},r.a.createElement(g,null))}var U={default:function(){return B.a},localeDropdown:function(){return A},search:function(){return V},docsVersion:function(){return a(269).default},docsVersionDropdown:function(){return a(270).default},doc:function(){return a(271).default}};function R(e){var t=e.type,a=Object(D.a)(e,["type"]),n=function(e){void 0===e&&(e="default");var t=U[e];if(!t)throw new Error("No NavbarItem component found for type="+e+".");return t()}(t);return r.a.createElement(n,a)}var F=a(250),H=a(252),X=a(61),z=a.n(X),G="right";var W=function(){var e,t=Object(m.useThemeConfig)(),a=t.navbar,o=a.items,l=a.hideOnScroll,i=a.style,s=t.colorMode.disableSwitch,u=Object(n.useState)(!1),d=u[0],f=u[1],h=Object(N.a)(),v=h.isDarkTheme,b=h.setLightTheme,k=h.setDarkTheme,E=S(l),O=E.navbarRef,j=E.isNavbarVisible;Object(L.a)(d);var y=Object(n.useCallback)((function(){f(!0)}),[f]),_=Object(n.useCallback)((function(){f(!1)}),[f]),C=Object(n.useCallback)((function(e){return e.target.checked?k():b()}),[b,k]),T=Object(I.a)();Object(n.useEffect)((function(){T===I.b.desktop&&f(!1)}),[T]);var D=o.some((function(e){return"search"===e.type})),B=function(e){return{leftItems:e.filter((function(e){var t;return"left"===(null!==(t=e.position)&&void 0!==t?t:G)})),rightItems:e.filter((function(e){var t;return"right"===(null!==(t=e.position)&&void 0!==t?t:G)}))}}(o),x=B.leftItems,A=B.rightItems;return r.a.createElement("nav",{ref:O,className:Object(c.a)("navbar","navbar--fixed-top",(e={"navbar--dark":"dark"===i,"navbar--primary":"primary"===i,"navbar-sidebar--show":d},e[z.a.navbarHideable]=l,e[z.a.navbarHidden]=l&&!j,e))},r.a.createElement("div",{className:"navbar__inner"},r.a.createElement("div",{className:"navbar__items"},null!=o&&0!==o.length&&r.a.createElement("button",{"aria-label":"Navigation bar toggle",className:"navbar__toggle",type:"button",tabIndex:0,onClick:y,onKeyDown:y},r.a.createElement(H.a,null)),r.a.createElement(F.a,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:Object(c.a)("navbar__title")}),x.map((function(e,t){return r.a.createElement(R,Object(p.a)({},e,{key:t}))}))),r.a.createElement("div",{className:"navbar__items navbar__items--right"},A.map((function(e,t){return r.a.createElement(R,Object(p.a)({},e,{key:t}))})),!s&&r.a.createElement(w,{className:z.a.displayOnlyInLargeViewport,"aria-label":"Dark mode toggle",checked:v,onChange:C}),!D&&r.a.createElement(g,null))),r.a.createElement("div",{role:"presentation",className:"navbar-sidebar__backdrop",onClick:_}),r.a.createElement("div",{className:"navbar-sidebar"},r.a.createElement("div",{className:"navbar-sidebar__brand"},r.a.createElement(F.a,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:"navbar__title",onClick:_}),!s&&d&&r.a.createElement(w,{"aria-label":"Dark mode toggle in sidebar",checked:v,onChange:C})),r.a.createElement("div",{className:"navbar-sidebar__items"},r.a.createElement("div",{className:"menu"},r.a.createElement("ul",{className:"menu__list"},o.map((function(e,t){return r.a.createElement(R,Object(p.a)({mobile:!0},e,{onClick:_,key:t}))})))))))},Y=a(233),K=a(234),Q=a(62),q=a.n(Q),J=a(251);function Z(e){var t=e.to,a=e.href,n=e.label,c=e.prependBaseUrlToHref,o=Object(D.a)(e,["to","href","label","prependBaseUrlToHref"]),l=Object(K.a)(t),i=Object(K.a)(a,{forcePrependBaseUrl:!0});return r.a.createElement(Y.a,Object(p.a)({className:"footer__link-item"},a?{href:c?i:a}:{to:l},o),n)}var $=function(e){var t=e.sources,a=e.alt;return r.a.createElement(J.a,{className:"footer__logo",alt:a,sources:t})};var ee=function(){var e=Object(m.useThemeConfig)().footer,t=e||{},a=t.copyright,n=t.links,o=void 0===n?[]:n,l=t.logo,i=void 0===l?{}:l,s={light:Object(K.a)(i.src),dark:Object(K.a)(i.srcDark||i.src)};return e?r.a.createElement("footer",{className:Object(c.a)("footer",{"footer--dark":"dark"===e.style})},r.a.createElement("div",{className:"container"},o&&o.length>0&&r.a.createElement("div",{className:"row footer__links"},o.map((function(e,t){return r.a.createElement("div",{key:t,className:"col footer__col"},null!=e.title?r.a.createElement("h4",{className:"footer__title"},e.title):null,null!=e.items&&Array.isArray(e.items)&&e.items.length>0?r.a.createElement("ul",{className:"footer__items"},e.items.map((function(e,t){return e.html?r.a.createElement("li",{key:t,className:"footer__item",dangerouslySetInnerHTML:{__html:e.html}}):r.a.createElement("li",{key:e.href||e.to,className:"footer__item"},r.a.createElement(Z,e))}))):null)}))),(i||a)&&r.a.createElement("div",{className:"footer__bottom text--center"},i&&(i.src||i.srcDark)&&r.a.createElement("div",{className:"margin-bottom--sm"},i.href?r.a.createElement(Y.a,{href:i.href,className:q.a.footerLogoLink},r.a.createElement($,{alt:i.alt,sources:s})):r.a.createElement($,{alt:i.alt,sources:s})),a?r.a.createElement("div",{className:"footer__copyright",dangerouslySetInnerHTML:{__html:a}}):null))):null},te=a(8),ae="light",ne="dark",re=function(e){return e===ne?ne:ae},ce=function(e){try{localStorage.setItem("theme",re(e))}catch(t){console.error(t)}},oe=function(){var e=Object(m.useThemeConfig)().colorMode,t=e.defaultMode,a=e.disableSwitch,r=e.respectPrefersColorScheme,c=Object(n.useState)(function(e){return te.a.canUseDOM?re(document.documentElement.getAttribute("data-theme")):re(e)}(t)),o=c[0],l=c[1],i=Object(n.useCallback)((function(){l(ae),ce(ae)}),[]),s=Object(n.useCallback)((function(){l(ne),ce(ne)}),[]);return Object(n.useEffect)((function(){document.documentElement.setAttribute("data-theme",re(o))}),[o]),Object(n.useEffect)((function(){if(!a)try{var e=localStorage.getItem("theme");null!==e&&l(re(e))}catch(t){console.error(t)}}),[l]),Object(n.useEffect)((function(){a&&!r||window.matchMedia("(prefers-color-scheme: dark)").addListener((function(e){var t=e.matches;l(t?ne:ae)}))}),[]),{isDarkTheme:o===ne,setLightTheme:i,setDarkTheme:s}},le=a(246);var ie=function(e){var t=oe(),a=t.isDarkTheme,n=t.setLightTheme,c=t.setDarkTheme;return r.a.createElement(le.a.Provider,{value:{isDarkTheme:a,setLightTheme:n,setDarkTheme:c}},e.children)},se="docusaurus.tab.",ue=function(){var e=Object(n.useState)({}),t=e[0],a=e[1],r=Object(n.useCallback)((function(e,t){try{localStorage.setItem("docusaurus.tab."+e,t)}catch(a){console.error(a)}}),[]);return Object(n.useEffect)((function(){try{for(var e={},t=0;t<localStorage.length;t+=1){var n=localStorage.key(t);if(n.startsWith(se))e[n.substring(se.length)]=localStorage.getItem(n)}a(e)}catch(r){console.error(r)}}),[]),{tabGroupChoices:t,setTabGroupChoices:function(e,t){a((function(a){var n;return Object.assign({},a,((n={})[e]=t,n))})),r(e,t)}}},de="docusaurus.announcement.dismiss",me="docusaurus.announcement.id",fe=function(){var e=Object(m.useThemeConfig)().announcementBar,t=Object(n.useState)(!0),a=t[0],r=t[1],c=Object(n.useCallback)((function(){localStorage.setItem(de,"true"),r(!0)}),[]);return Object(n.useEffect)((function(){if(e){var t=e.id,a=localStorage.getItem(me);"annoucement-bar"===a&&(a="announcement-bar");var n=t!==a;localStorage.setItem(me,t),n&&localStorage.setItem(de,"false"),(n||"false"===localStorage.getItem(de))&&r(!1)}}),[]),{isAnnouncementBarClosed:a,closeAnnouncementBar:c}},he=a(244);var ve=function(e){var t=ue(),a=t.tabGroupChoices,n=t.setTabGroupChoices,c=fe(),o=c.isAnnouncementBarClosed,l=c.closeAnnouncementBar;return r.a.createElement(he.a.Provider,{value:{tabGroupChoices:a,setTabGroupChoices:n,isAnnouncementBarClosed:o,closeAnnouncementBar:l}},e.children)};function be(e){var t=e.children;return r.a.createElement(ie,null,r.a.createElement(ve,null,r.a.createElement(m.DocsPreferredVersionContextProvider,null,t)))}var pe=a(25);function ge(e){var t=e.locale,a=e.version,n=e.tag;return r.a.createElement(pe.a,null,t&&r.a.createElement("meta",{name:"docusaurus_locale",content:""+t}),a&&r.a.createElement("meta",{name:"docusaurus_version",content:a}),n&&r.a.createElement("meta",{name:"docusaurus_tag",content:n}))}var ke=a(253);function Ee(){var e=Object(O.default)().i18n,t=e.defaultLocale,a=e.locales,n=Object(m.useAlternatePageUtils)();return r.a.createElement(pe.a,null,a.map((function(e){return r.a.createElement("link",{key:e,rel:"alternate",href:n.createUrl({locale:e,fullyQualified:!0}),hrefLang:e})})),r.a.createElement("link",{rel:"alternate",href:n.createUrl({locale:t,fullyQualified:!0}),hrefLang:"x-default"}))}function Oe(e){var t=e.permalink,a=Object(O.default)().siteConfig.url,n=function(){var e=Object(O.default)().siteConfig.url,t=Object(l.useLocation)().pathname;return e+Object(K.a)(t)}(),c=t?""+a+t:n;return r.a.createElement(pe.a,null,r.a.createElement("meta",{property:"og:url",content:c}),r.a.createElement("link",{rel:"canonical",href:c}))}function je(e){var t=Object(O.default)(),a=t.siteConfig,n=t.i18n,c=n.currentLocale,o=n.localeConfigs,l=a.favicon,i=a.themeConfig,s=i.image,u=i.metadatas,d=e.title,f=e.description,h=e.image,v=e.keywords,b=e.searchMetadatas,g=Object(K.a)(l),k=c,E=o[c].direction;return r.a.createElement(r.a.Fragment,null,r.a.createElement(pe.a,null,r.a.createElement("html",{lang:k,dir:E}),l&&r.a.createElement("link",{rel:"shortcut icon",href:g})),r.a.createElement(ke.a,{title:d,description:f,keywords:v,image:h||s}),r.a.createElement(Oe,null),r.a.createElement(Ee,null),r.a.createElement(ge,Object(p.a)({tag:m.DEFAULT_SEARCH_TAG,locale:c},b)),r.a.createElement(pe.a,null,u.map((function(e,t){return r.a.createElement("meta",Object(p.a)({key:"metadata_"+t},e))}))))}a(63);var ye=function(){Object(n.useEffect)((function(){var e="navigation-with-keyboard";function t(t){"keydown"===t.type&&"Tab"===t.key&&document.body.classList.add(e),"mousedown"===t.type&&document.body.classList.remove(e)}return document.addEventListener("keydown",t),document.addEventListener("mousedown",t),function(){document.body.classList.remove(e),document.removeEventListener("keydown",t),document.removeEventListener("mousedown",t)}}),[])};a(64);t.a=function(e){var t=e.children,a=e.noFooter,n=e.wrapperClassName;return ye(),r.a.createElement(be,null,r.a.createElement(je,e),r.a.createElement(d,null),r.a.createElement(b,null),r.a.createElement(W,null),r.a.createElement("div",{className:Object(c.a)("main-wrapper",n)},t),!a&&r.a.createElement(ee,null))}},237:function(e,t,a){"use strict";var n=a(0),r=a(246);t.a=function(){var e=Object(n.useContext)(r.a);if(null==e)throw new Error("`useThemeContext` is used outside of `Layout` Component. See https://v2.docusaurus.io/docs/api/themes/configuration#usethemecontext.");return e}},243:function(e,t,a){"use strict";var n=a(0),r=a(244);t.a=function(){var e=Object(n.useContext)(r.a);if(null==e)throw new Error("`useUserPreferencesContext` is used outside of `Layout` Component.");return e}},244:function(e,t,a){"use strict";var n=a(0),r=Object(n.createContext)(void 0);t.a=r},245:function(e,t,a){var n;!function(){"use strict";var a={}.hasOwnProperty;function r(){for(var e=[],t=0;t<arguments.length;t++){var n=arguments[t];if(n){var c=typeof n;if("string"===c||"number"===c)e.push(n);else if(Array.isArray(n)&&n.length){var o=r.apply(null,n);o&&e.push(o)}else if("object"===c)for(var l in n)a.call(n,l)&&n[l]&&e.push(l)}}return e.join(" ")}e.exports?(r.default=r,e.exports=r):void 0===(n=function(){return r}.apply(t,[]))||(e.exports=n)}()},246:function(e,t,a){"use strict";var n=a(0),r=a.n(n).a.createContext(void 0);t.a=r},247:function(e,t,a){"use strict";var n=a(0),r=a(8),c=function(){return{scrollX:r.a.canUseDOM?window.pageXOffset:0,scrollY:r.a.canUseDOM?window.pageYOffset:0}};t.a=function(e,t){void 0===t&&(t=[]);var a=Object(n.useState)(c()),r=a[0],o=a[1],l=function(){var t=c();o(t),e&&e(t)};return Object(n.useEffect)((function(){var e={passive:!0};return window.addEventListener("scroll",l,e),function(){return window.removeEventListener("scroll",l,e)}}),t),r}},248:function(e,t,a){"use strict";var n=a(0);t.a=function(e){void 0===e&&(e=!0),Object(n.useEffect)((function(){return document.body.style.overflow=e?"hidden":"visible",function(){document.body.style.overflow="visible"}}),[e])}},249:function(e,t,a){"use strict";a.d(t,"b",(function(){return c}));var n=a(0),r=a(8),c={desktop:"desktop",mobile:"mobile"};t.a=function(){var e=r.a.canUseDOM;function t(){if(e)return window.innerWidth>996?c.desktop:c.mobile}var a=Object(n.useState)(t),o=a[0],l=a[1];return Object(n.useEffect)((function(){if(e)return window.addEventListener("resize",a),function(){return window.removeEventListener("resize",a)};function a(){l(t())}}),[]),o}},250:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(233),i=a(251),s=a(234),u=a(16),d=a(229);t.a=function(e){var t=Object(u.default)().isClient,a=Object(d.useThemeConfig)().navbar,c=a.title,m=a.logo,f=void 0===m?{src:""}:m,h=e.imageClassName,v=e.titleClassName,b=Object(r.a)(e,["imageClassName","titleClassName"]),p=Object(s.a)(f.href||"/"),g={light:Object(s.a)(f.src),dark:Object(s.a)(f.srcDark||f.src)};return o.a.createElement(l.a,Object(n.a)({to:p},b,f.target&&{target:f.target}),f.src&&o.a.createElement(i.a,{key:t,className:h,sources:g,alt:f.alt||c||"Logo"}),null!=c&&o.a.createElement("strong",{className:v},c))}},251:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(230),i=a(16),s=a(237),u=a(60),d=a.n(u);t.a=function(e){var t=Object(i.default)().isClient,a=Object(s.a)().isDarkTheme,c=e.sources,u=e.className,m=e.alt,f=void 0===m?"":m,h=Object(r.a)(e,["sources","className","alt"]),v=t?a?["dark"]:["light"]:["light","dark"];return o.a.createElement(o.a.Fragment,null,v.map((function(e){return o.a.createElement("img",Object(n.a)({key:e,src:c[e],alt:f,className:Object(l.a)(d.a.themedImage,d.a["themedImage--"+e],u)},h))})))}},252:function(e,t,a){"use strict";var n=a(3),r=a(7),c=a(0),o=a.n(c);t.a=function(e){var t=e.width,a=void 0===t?30:t,c=e.height,l=void 0===c?30:c,i=e.className,s=Object(r.a)(e,["width","height","className"]);return o.a.createElement("svg",Object(n.a)({"aria-label":"Menu",className:i,width:a,height:l,viewBox:"0 0 30 30",role:"img",focusable:"false"},s),o.a.createElement("title",null,"Menu"),o.a.createElement("path",{stroke:"currentColor",strokeLinecap:"round",strokeMiterlimit:"10",strokeWidth:"2",d:"M4 7h22M4 15h22M4 23h22"}))}},265:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var a=arguments[t];for(var n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n])}return e},r=function(){function e(e,t){for(var a=0;a<t.length;a++){var n=t[a];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,a,n){return a&&e(t.prototype,a),n&&e(t,n),t}}(),c=a(0),o=m(c),l=m(a(245)),i=m(a(1)),s=m(a(266)),u=m(a(267)),d=a(268);function m(e){return e&&e.__esModule?e:{default:e}}var f=function(e){function t(e){!function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}(this,t);var a=function(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e));return a.handleClick=a.handleClick.bind(a),a.handleTouchStart=a.handleTouchStart.bind(a),a.handleTouchMove=a.handleTouchMove.bind(a),a.handleTouchEnd=a.handleTouchEnd.bind(a),a.handleFocus=a.handleFocus.bind(a),a.handleBlur=a.handleBlur.bind(a),a.previouslyChecked=!(!e.checked&&!e.defaultChecked),a.state={checked:!(!e.checked&&!e.defaultChecked),hasFocus:!1},a}return function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}(t,e),r(t,[{key:"componentDidUpdate",value:function(e){e.checked!==this.props.checked&&this.setState({checked:!!this.props.checked})}},{key:"handleClick",value:function(e){if(!this.props.disabled){var t=this.input;if(e.target!==t&&!this.moved)return this.previouslyChecked=t.checked,e.preventDefault(),t.focus(),void t.click();var a=this.props.hasOwnProperty("checked")?this.props.checked:t.checked;this.setState({checked:a})}}},{key:"handleTouchStart",value:function(e){this.props.disabled||(this.startX=(0,d.pointerCoord)(e).x,this.activated=!0)}},{key:"handleTouchMove",value:function(e){if(this.activated&&(this.moved=!0,this.startX)){var t=(0,d.pointerCoord)(e).x;this.state.checked&&t+15<this.startX?(this.setState({checked:!1}),this.startX=t,this.activated=!0):t-15>this.startX&&(this.setState({checked:!0}),this.startX=t,this.activated=t<this.startX+5)}}},{key:"handleTouchEnd",value:function(e){if(this.moved){var t=this.input;if(e.preventDefault(),this.startX){var a=(0,d.pointerCoord)(e).x;!0===this.previouslyChecked&&this.startX+4>a?this.previouslyChecked!==this.state.checked&&(this.setState({checked:!1}),this.previouslyChecked=this.state.checked,t.click()):this.startX-4<a&&this.previouslyChecked!==this.state.checked&&(this.setState({checked:!0}),this.previouslyChecked=this.state.checked,t.click()),this.activated=!1,this.startX=null,this.moved=!1}}}},{key:"handleFocus",value:function(e){var t=this.props.onFocus;t&&t(e),this.setState({hasFocus:!0})}},{key:"handleBlur",value:function(e){var t=this.props.onBlur;t&&t(e),this.setState({hasFocus:!1})}},{key:"getIcon",value:function(e){var a=this.props.icons;return a?void 0===a[e]?t.defaultProps.icons[e]:a[e]:null}},{key:"render",value:function(){var e=this,t=this.props,a=t.className,r=(t.icons,function(e,t){var a={};for(var n in e)t.indexOf(n)>=0||Object.prototype.hasOwnProperty.call(e,n)&&(a[n]=e[n]);return a}(t,["className","icons"])),c=(0,l.default)("react-toggle",{"react-toggle--checked":this.state.checked,"react-toggle--focus":this.state.hasFocus,"react-toggle--disabled":this.props.disabled},a);return o.default.createElement("div",{className:c,onClick:this.handleClick,onTouchStart:this.handleTouchStart,onTouchMove:this.handleTouchMove,onTouchEnd:this.handleTouchEnd},o.default.createElement("div",{className:"react-toggle-track"},o.default.createElement("div",{className:"react-toggle-track-check"},this.getIcon("checked")),o.default.createElement("div",{className:"react-toggle-track-x"},this.getIcon("unchecked"))),o.default.createElement("div",{className:"react-toggle-thumb"}),o.default.createElement("input",n({},r,{ref:function(t){e.input=t},onFocus:this.handleFocus,onBlur:this.handleBlur,className:"react-toggle-screenreader-only",type:"checkbox"})))}}]),t}(c.PureComponent);t.default=f,f.displayName="Toggle",f.defaultProps={icons:{checked:o.default.createElement(s.default,null),unchecked:o.default.createElement(u.default,null)}},f.propTypes={checked:i.default.bool,disabled:i.default.bool,defaultChecked:i.default.bool,onChange:i.default.func,onFocus:i.default.func,onBlur:i.default.func,className:i.default.string,name:i.default.string,value:i.default.string,id:i.default.string,"aria-labelledby":i.default.string,"aria-label":i.default.string,icons:i.default.oneOfType([i.default.bool,i.default.shape({checked:i.default.node,unchecked:i.default.node})])}},266:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n,r=a(0),c=(n=r)&&n.__esModule?n:{default:n};t.default=function(){return c.default.createElement("svg",{width:"14",height:"11",viewBox:"0 0 14 11"},c.default.createElement("path",{d:"M11.264 0L5.26 6.004 2.103 2.847 0 4.95l5.26 5.26 8.108-8.107L11.264 0",fill:"#fff",fillRule:"evenodd"}))}},267:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n,r=a(0),c=(n=r)&&n.__esModule?n:{default:n};t.default=function(){return c.default.createElement("svg",{width:"10",height:"10",viewBox:"0 0 10 10"},c.default.createElement("path",{d:"M9.9 2.12L7.78 0 4.95 2.828 2.12 0 0 2.12l2.83 2.83L0 7.776 2.123 9.9 4.95 7.07 7.78 9.9 9.9 7.776 7.072 4.95 9.9 2.12",fill:"#fff",fillRule:"evenodd"}))}},268:function(e,t,a){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.pointerCoord=function(e){if(e){var t=e.changedTouches;if(t&&t.length>0){var a=t[0];return{x:a.clientX,y:a.clientY}}var n=e.pageX;if(void 0!==n)return{x:n,y:e.pageY}}return{x:0,y:0}}},269:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return u}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(235),i=a(232),s=a(229);function u(e){var t,a=e.label,c=e.to,u=e.docsPluginId,d=Object(r.a)(e,["label","to","docsPluginId"]),m=Object(i.useActiveVersion)(u),f=Object(s.useDocsPreferredVersion)(u).preferredVersion,h=Object(i.useLatestVersion)(u),v=null!==(t=null!=m?m:f)&&void 0!==t?t:h,b=null!=a?a:v.label,p=null!=c?c:function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))}(v).path;return o.a.createElement(l.a,Object(n.a)({},d,{label:b,to:p}))}},270:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return d}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(235),i=a(232),s=a(229),u=function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))};function d(e){var t,a,c=e.mobile,d=e.docsPluginId,m=e.dropdownActiveClassDisabled,f=e.dropdownItemsBefore,h=e.dropdownItemsAfter,v=Object(r.a)(e,["mobile","docsPluginId","dropdownActiveClassDisabled","dropdownItemsBefore","dropdownItemsAfter"]),b=Object(i.useActiveDocContext)(d),p=Object(i.useVersions)(d),g=Object(i.useLatestVersion)(d),k=Object(s.useDocsPreferredVersion)(d),E=k.preferredVersion,O=k.savePreferredVersionName;var j=null!==(t=null!==(a=b.activeVersion)&&void 0!==a?a:E)&&void 0!==t?t:g,y=c?"Versions":j.label,_=c?void 0:u(j).path;return o.a.createElement(l.a,Object(n.a)({},v,{mobile:c,label:y,to:_,items:function(){var e=p.map((function(e){var t=(null==b?void 0:b.alternateDocVersions[e.name])||u(e);return{isNavLink:!0,label:e.label,to:t.path,isActive:function(){return e===(null==b?void 0:b.activeVersion)},onClick:function(){O(e.name)}}})),t=[].concat(f,e,h);if(!(t.length<=1))return t}(),isActive:m?function(){return!1}:void 0}))}},271:function(e,t,a){"use strict";a.r(t),a.d(t,"default",(function(){return d}));var n=a(3),r=a(7),c=a(0),o=a.n(c),l=a(235),i=a(232),s=a(230),u=a(229);function d(e){var t,a,c=e.docId,d=e.activeSidebarClassName,m=e.label,f=e.docsPluginId,h=Object(r.a)(e,["docId","activeSidebarClassName","label","docsPluginId"]),v=Object(i.useActiveDocContext)(f),b=v.activeVersion,p=v.activeDoc,g=Object(u.useDocsPreferredVersion)(f).preferredVersion,k=Object(i.useLatestVersion)(f),E=null!==(t=null!=b?b:g)&&void 0!==t?t:k,O=E.docs.find((function(e){return e.id===c}));if(!O)throw new Error("DocNavbarItem: couldn't find any doc with id="+c+" in version "+E.name+".\nAvailable docIds=\n- "+E.docs.join("\n- "));return o.a.createElement(l.a,Object(n.a)({exact:!0},h,{className:Object(s.a)(h.className,(a={},a[d]=p&&p.sidebar===O.sidebar,a)),label:null!=m?m:O.id,to:O.path}))}}}]);
\ No newline at end of file
diff --git a/assets/js/2.89c52fba.js.LICENSE.txt b/assets/js/2.f7e8da3d.js.LICENSE.txt
similarity index 100%
rename from assets/js/2.89c52fba.js.LICENSE.txt
rename to assets/js/2.f7e8da3d.js.LICENSE.txt
diff --git a/assets/js/21ee0bc4.17500718.js b/assets/js/21ee0bc4.03721b9e.js
similarity index 96%
rename from assets/js/21ee0bc4.17500718.js
rename to assets/js/21ee0bc4.03721b9e.js
index d2f45f11..7cc1ea50 100644
--- a/assets/js/21ee0bc4.17500718.js
+++ b/assets/js/21ee0bc4.03721b9e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[22],{232:function(e,t,r){"use strict";r.d(t,"a",(function(){return u})),r.d(t,"b",(function(){return m}));var n=r(0),i=r.n(n);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){o(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function p(e,t){if(null==e)return{};var r,n,i=function(e,t){if(null==e)return{};var r,n,i={},o=Object.keys(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,a=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(r),f=n,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return r?i.a.createElement(m,l(l({ref:t},c),{},{components:r})):i.a.createElement(m,l({ref:t},c))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,a=new Array(o);a[0]=f;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<o;c++)a[c]=r[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,r)}f.displayName="MDXCreateElement"},92:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return l})),r.d(t,"toc",(function(){return p})),r.d(t,"default",(function(){return s}));var n=r(3),i=r(7),o=(r(0),r(232)),a={id:"pipeline_importer",title:"Importer",sidebar_label:"Importer"},l={unversionedId:"pipeline_importer",id:"version-1.2.6/pipeline_importer",isDocsHomePage:!1,title:"Importer",description:"Importer",source:"@site/versioned_docs/version-1.2.6/pipeline_importer.md",slug:"/pipeline_importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_importer.md",version:"1.2.6",sidebar_label:"Importer",sidebar:"version-1.2.6/Pipeline",previous:{title:"Pipeline",permalink:"/netsage-pipeline/docs/1.2.6/pipeline"},next:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash"}},p=[{value:"Importer",id:"importer",children:[]},{value:"Importer",id:"importer-1",children:[{value:"Configuration",id:"configuration",children:[]},{value:"Internals",id:"internals",children:[]}]}],c={toc:p};function s(e){var t=e.components,r=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(n.a)({},c,r,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."),Object(o.b)("h2",{id:"importer-1"},"Importer"),Object(o.b)("p",null,"NOTE: Importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"configuration files for the importer are ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_netflow_importer.xml")," and ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," in ",Object(o.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifer/"),". Comments in the files briefly describe the options."),Object(o.b)("p",null,"Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. "),Object(o.b)("h3",{id:"internals"},"Internals"),Object(o.b)("p",null,"The importer uses the nfdump command with -a to aggregate within the file, and -L ",Object(o.b)("inlineCode",{parentName:"p"},"threshold")," to throw out any flows under a flow size threshold. "),Object(o.b)("p",null,"For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.\nA ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages."),Object(o.b)("p",null,"All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows."))}s.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[23],{228:function(e,t,r){"use strict";r.d(t,"a",(function(){return u})),r.d(t,"b",(function(){return m}));var n=r(0),i=r.n(n);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){o(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function p(e,t){if(null==e)return{};var r,n,i=function(e,t){if(null==e)return{};var r,n,i={},o=Object.keys(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,a=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(r),f=n,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return r?i.a.createElement(m,l(l({ref:t},c),{},{components:r})):i.a.createElement(m,l({ref:t},c))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,a=new Array(o);a[0]=f;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<o;c++)a[c]=r[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,r)}f.displayName="MDXCreateElement"},93:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return l})),r.d(t,"toc",(function(){return p})),r.d(t,"default",(function(){return s}));var n=r(3),i=r(7),o=(r(0),r(228)),a={id:"pipeline_importer",title:"Importer",sidebar_label:"Importer"},l={unversionedId:"pipeline_importer",id:"version-1.2.6/pipeline_importer",isDocsHomePage:!1,title:"Importer",description:"Importer",source:"@site/versioned_docs/version-1.2.6/pipeline_importer.md",slug:"/pipeline_importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_importer.md",version:"1.2.6",sidebar_label:"Importer",sidebar:"version-1.2.6/Pipeline",previous:{title:"Pipeline",permalink:"/netsage-pipeline/docs/1.2.6/pipeline"},next:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash"}},p=[{value:"Importer",id:"importer",children:[]},{value:"Importer",id:"importer-1",children:[{value:"Configuration",id:"configuration",children:[]},{value:"Internals",id:"internals",children:[]}]}],c={toc:p};function s(e){var t=e.components,r=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(n.a)({},c,r,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."),Object(o.b)("h2",{id:"importer-1"},"Importer"),Object(o.b)("p",null,"NOTE: Importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"configuration files for the importer are ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_netflow_importer.xml")," and ",Object(o.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," in ",Object(o.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifer/"),". Comments in the files briefly describe the options."),Object(o.b)("p",null,"Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. "),Object(o.b)("h3",{id:"internals"},"Internals"),Object(o.b)("p",null,"The importer uses the nfdump command with -a to aggregate within the file, and -L ",Object(o.b)("inlineCode",{parentName:"p"},"threshold")," to throw out any flows under a flow size threshold. "),Object(o.b)("p",null,"For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.\nA ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages."),Object(o.b)("p",null,"All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows."))}s.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/22b7278f.fe1cf5a5.js b/assets/js/22b7278f.58242775.js
similarity index 97%
rename from assets/js/22b7278f.fe1cf5a5.js
rename to assets/js/22b7278f.58242775.js
index cb922ccd..588b4785 100644
--- a/assets/js/22b7278f.fe1cf5a5.js
+++ b/assets/js/22b7278f.58242775.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[23],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},93:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(232)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/docs/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/next/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/intro.md",version:"current",sidebar_label:"Intro",sidebar:"Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[24],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},94:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(228)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/docs/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/next/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/intro.md",version:"current",sidebar_label:"Intro",sidebar:"Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/22d86c11.0650c59f.js b/assets/js/22d86c11.2a981864.js
similarity index 99%
rename from assets/js/22d86c11.0650c59f.js
rename to assets/js/22d86c11.2a981864.js
index bc1546b9..c8fddba5 100644
--- a/assets/js/22d86c11.0650c59f.js
+++ b/assets/js/22d86c11.2a981864.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[24],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(i,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,l(l({ref:t},c),{},{components:n})):a.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},94:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.11/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",version:"1.2.11",sidebar_label:"Docker Advanced Options",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[25],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(i,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,l(l({ref:t},c),{},{components:n})):a.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},95:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.11/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",version:"1.2.11",sidebar_label:"Docker Advanced Options",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/3179f6dc.91efb769.js b/assets/js/3179f6dc.bedc2974.js
similarity index 96%
rename from assets/js/3179f6dc.91efb769.js
rename to assets/js/3179f6dc.bedc2974.js
index e1dcd005..012580b6 100644
--- a/assets/js/3179f6dc.91efb769.js
+++ b/assets/js/3179f6dc.bedc2974.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[25],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,f=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},95:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),i=(n(0),n(232)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.5/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal",source:"@site/versioned_docs/version-1.2.5/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/choosing.md",version:"1.2.5",sidebar_label:"Choose Install",sidebar:"version-1.2.5/Pipeline",previous:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker"},next:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install"}},s=[{value:"BareMetal",id:"baremetal",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal"},"BareMetal"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"docker_install"},"Docker")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"install"},"BareMetal"))))}u.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[26],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,f=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},96:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.5/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal",source:"@site/versioned_docs/version-1.2.5/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/choosing.md",version:"1.2.5",sidebar_label:"Choose Install",sidebar:"version-1.2.5/Pipeline",previous:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker"},next:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install"}},s=[{value:"BareMetal",id:"baremetal",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal"},"BareMetal"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"docker_install"},"Docker")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"install"},"BareMetal"))))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/3309db24.62c09093.js b/assets/js/3309db24.f6153687.js
similarity index 99%
rename from assets/js/3309db24.62c09093.js
rename to assets/js/3309db24.f6153687.js
index ecd0ff3e..790921b0 100644
--- a/assets/js/3309db24.62c09093.js
+++ b/assets/js/3309db24.f6153687.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[26],{232:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"},96:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.7/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.7/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/elastic_search.md",version:"1.2.7",sidebar_label:"Elasticsearch",sidebar:"version-1.2.7/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"sflow, netflow, or tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type (Circuit, Regional Network, etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_namess"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'A "Project" that the resource is part')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"..."),Object(l.b)("td",{parentName:"tr",align:null},"An object containing project info")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values, otherwise they are the CAIDA or MaxMind GeoIP values..")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null})),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[27],{228:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"},97:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.7/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.7/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/elastic_search.md",version:"1.2.7",sidebar_label:"Elasticsearch",sidebar:"version-1.2.7/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"sflow, netflow, or tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type (Circuit, Regional Network, etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_namess"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'A "Project" that the resource is part')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"..."),Object(l.b)("td",{parentName:"tr",align:null},"An object containing project info")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values, otherwise they are the CAIDA or MaxMind GeoIP values..")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null})),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/33f6b1c9.bd4095bc.js b/assets/js/33f6b1c9.798766ac.js
similarity index 97%
rename from assets/js/33f6b1c9.bd4095bc.js
rename to assets/js/33f6b1c9.798766ac.js
index 77b36286..05b70637 100644
--- a/assets/js/33f6b1c9.bd4095bc.js
+++ b/assets/js/33f6b1c9.798766ac.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[27],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},97:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(232)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.8/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.8/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/intro.md",version:"1.2.8",sidebar_label:"Intro",sidebar:"version-1.2.8/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/pipeline/importer"},"Doc"),", ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},"Doc"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[28],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},98:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(228)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.8/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.8/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/intro.md",version:"1.2.8",sidebar_label:"Intro",sidebar:"version-1.2.8/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/pipeline/importer"},"Doc"),", ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},"Doc"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/358e6759.d9fbc6e9.js b/assets/js/358e6759.19972cb0.js
similarity index 97%
rename from assets/js/358e6759.d9fbc6e9.js
rename to assets/js/358e6759.19972cb0.js
index ee64c619..bb4a1650 100644
--- a/assets/js/358e6759.d9fbc6e9.js
+++ b/assets/js/358e6759.19972cb0.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[28],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},98:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.7/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.7/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_env.md",version:"1.2.7"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},'sflowSensorName="my sflow sensor name"\nnetflowSensorName="my netflow sensor name"\n')),Object(r.b)("p",null,"Simply change the names to unique identifiers and you're good to go. (Use quotes if the names have spaces.) "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and can be used in visualizations. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet Sflow" and "RNDNet Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used."),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.'),Object(r.b)("li",{parentName:"ul"},"If you're not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don't start up either collector.")),Object(r.b)("p",null,"Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  Use the default ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit")," for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[29],{228:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},99:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.7/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.7/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_env.md",version:"1.2.7"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},'sflowSensorName="my sflow sensor name"\nnetflowSensorName="my netflow sensor name"\n')),Object(r.b)("p",null,"Simply change the names to unique identifiers and you're good to go. (Use quotes if the names have spaces.) "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and can be used in visualizations. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet Sflow" and "RNDNet Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used."),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.'),Object(r.b)("li",{parentName:"ul"},"If you're not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don't start up either collector.")),Object(r.b)("p",null,"Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  Use the default ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit")," for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/38962575.8edb151f.js b/assets/js/38962575.8edb151f.js
new file mode 100644
index 00000000..6e871691
--- /dev/null
+++ b/assets/js/38962575.8edb151f.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[30],{100:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"version-1.2.10/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",version:"1.2.10",sidebar_label:"Docker Upgrading",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Override Files",id:"checkupdate-override-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-override-files"},"Check/Update Override Files"),Object(a.b)("p",null,"Occasionally, the required version of Docker or nfdump may change, which will necessitate editing your override and/or env files."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," to see if a new version of Docker is required. Look for, eg, ",Object(a.b)("inlineCode",{parentName:"p"},'version: "3.7"')," at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),'. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.10). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this step!  Pull new images from Docker Hub. This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To save space, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/38962575.cec7a9ef.js b/assets/js/38962575.cec7a9ef.js
deleted file mode 100644
index 5ac1a479..00000000
--- a/assets/js/38962575.cec7a9ef.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[29],{232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"},99:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"version-1.2.10/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",version:"1.2.10",sidebar_label:"Docker Upgrading",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Override Files",id:"checkupdate-override-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-override-files"},"Check/Update Override Files"),Object(a.b)("p",null,"Occasionally, the required version of Docker or nfdump may change, which will necessitate editing your override and/or env files."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," to see if a new version of Docker is required. Look for, eg, ",Object(a.b)("inlineCode",{parentName:"p"},'version: "3.7"')," at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),'. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.10). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this step!  Pull new images from Docker Hub. This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To save space, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")))}d.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/38d1fe39.6ca6725c.js b/assets/js/38d1fe39.e28e353a.js
similarity index 99%
rename from assets/js/38d1fe39.6ca6725c.js
rename to assets/js/38d1fe39.e28e353a.js
index 3908d2a3..47559be3 100644
--- a/assets/js/38d1fe39.6ca6725c.js
+++ b/assets/js/38d1fe39.e28e353a.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[30],{100:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return b})),a.d(t,"toc",(function(){return c})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(232)),i={id:"pipeline_logstash",title:"Pipeline Logstash",sidebar_label:"Logstash"},b={unversionedId:"pipeline_logstash",id:"version-1.2.6/pipeline_logstash",isDocsHomePage:!1,title:"Pipeline Logstash",description:"Logstash",source:"@site/versioned_docs/version-1.2.6/pipeline_logstash.md",slug:"/pipeline_logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_logstash.md",version:"1.2.6",sidebar_label:"Logstash",sidebar:"version-1.2.6/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer"},next:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[{value:"ES fields",id:"es-fields",children:[]},{value:"Developer fields",id:"developer-fields",children:[]},{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly)",id:"source-fields-destination-fields-similarly",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly)",id:"source-science-registry-fields--destination-fields-similarly",children:[]},{value:"&quot;Preferred&quot; fields",id:"preferred-fields",children:[]},{value:"Values",id:"values",children:[]},{value:"Tstat Values",id:"tstat-values",children:[]}]}],o={toc:c};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(l.b)("wrapper",Object(n.a)({},o,a,{components:t,mdxType:"MDXLayout"}),Object(l.b)("h1",{id:"logstash"},"Logstash"),Object(l.b)("p",null,"These Logstash config files are in /etc/logstash/conf.d/"),Object(l.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(l.b)("p",null,"The following steps are defined for logstash:"),Object(l.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(l.b)("p",null,'Reads flows from a rabbitmq queue. (".disabled" can be removed from other input configs to get flows from other sources.)'),Object(l.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(l.b)("p",null,"Drops flows to or from private IP addresses;\nadds @ingest_time (this is mainly for developers);\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions"),Object(l.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(l.b)("p",null,"Adds a unique id based on the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This ends up being called meta.id."),Object(l.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(l.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(l.b)("p",null,"Notes: By default, 5-minute nfcapd files are assumed, and if less than 10.5 min have passed between the start of the current flow and the start of the last matching one, stitch the two together."),Object(l.b)("p",null,"Your logstash pipeline can have only 1 worker or aggregation is not going to work! "),Object(l.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(l.b)("p",null,'If the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast";\nqueries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes. '),Object(l.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(l.b)("p",null,"Normally, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, try getting an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(l.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(l.b)("p",null,"Uses the ASN determined previously to get the organization name from the prepared CAIDA lookup file."),Object(l.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(l.b)("p",null,"Search (optional) lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(l.b)("p",null,"Notes: These lookup tables are not stored in github."),Object(l.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(l.b)("p",null,"Uses a fake geoip database containing Science Registry information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves scireg fields we don't need to save to elasticsearch."),Object(l.b)("p",null,"Notes: The science registry fake geoip database can be downloaded from scienceregistry.grnoc.iu.edu via wget in a cron job. "),Object(l.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(l.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them. "),Object(l.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(l.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(l.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(l.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the preferred_organization and preferred_location fields."),Object(l.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(l.b)("p",null,"Sets additional quick and easy fields.  Currently we have:\n",Object(l.b)("em",{parentName:"p"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)\n"),"sensor_type    = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)\n",Object(l.b)("em",{parentName:"p"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)\n"),"is_network_testing = yes, no  (yes if discipline = CS.Network Testing and Monitoring or port = 5001, 5101, or 5201)"),Object(l.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(l.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(l.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(l.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(l.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(l.b)("p",null,'Sends results to a rabbitmq queue (".disabled" can be removed from other output configs to send flows to other places)'),Object(l.b)("h3",{id:"final-stage"},"Final Stage"),Object(l.b)("p",null,"In our case, OmniSOC manages the last stage. Their logstash instance reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-",Object(l.b)("em",{parentName:"p"}," (or om-ns-ilight-"),", etc).  "),Object(l.b)("p",null,"This can be easily replicated with the following configuration though you'll need one for each feed/index."),Object(l.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(l.b)("pre",null,Object(l.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(l.b)("p",null,"Once the data is published in elastic, you can use the ",Object(l.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboard")," to visualize the data."),Object(l.b)("h2",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("h3",{id:"es-fields"},"ES fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06."),Object(l.b)("td",{parentName:"tr",align:null},"equivalent to an sql table")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"document id, set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))),Object(l.b)("h3",{id:"developer-fields"},"Developer fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"type"),Object(l.b)("td",{parentName:"tr",align:null},"flow"),Object(l.b)("td",{parentName:"tr",align:null},'Always "flow" for us. Other types may be "macy", etc.')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@injest_time"),Object(l.b)("td",{parentName:"tr",align:null},"2020-06-09T21:51:57.059Z"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline (99-outputs.conf)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @injest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.")))),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned flow id")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"Sflow, Netflow, or Tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor group")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor type")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly"},"Source Fields (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"ASN of the IP from geoip ASN database or the ASN from the flow header")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS of the IP from geoip ASN database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly"},"Source Science Registry Fields  (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford - ImageNet"),Object(l.b)("td",{parentName:"tr",align:null},"Resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"-"),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"CS. Intelligent Systems"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"."),Object(l.b)("td",{parentName:"tr",align:null},"Can be an array of projects ","[we may change this field name soon]")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's location, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"preferred-fields"},'"Preferred" fields'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Otherwise, they are the geoip values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"values"},"Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-values"},"Tstat Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))))}s.isMDXComponent=!0},232:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function b(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){l(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function c(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var o=r.a.createContext({}),s=function(e){var t=r.a.useContext(o),a=t;return e&&(a="function"==typeof e?e(t):b(b({},t),e)),a},d=function(e){var t=s(e.components);return r.a.createElement(o.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,o=c(e,["components","mdxType","originalType","parentName"]),d=s(a),m=n,u=d["".concat(i,".").concat(m)]||d[m]||p[m]||l;return a?r.a.createElement(u,b(b({ref:t},o),{},{components:a})):r.a.createElement(u,b({ref:t},o))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,i=new Array(l);i[0]=m;var b={};for(var c in t)hasOwnProperty.call(t,c)&&(b[c]=t[c]);b.originalType=e,b.mdxType="string"==typeof e?e:n,i[1]=b;for(var o=2;o<l;o++)i[o]=a[o];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[31],{101:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return b})),a.d(t,"toc",(function(){return c})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(228)),i={id:"pipeline_logstash",title:"Pipeline Logstash",sidebar_label:"Logstash"},b={unversionedId:"pipeline_logstash",id:"version-1.2.6/pipeline_logstash",isDocsHomePage:!1,title:"Pipeline Logstash",description:"Logstash",source:"@site/versioned_docs/version-1.2.6/pipeline_logstash.md",slug:"/pipeline_logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_logstash.md",version:"1.2.6",sidebar_label:"Logstash",sidebar:"version-1.2.6/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer"},next:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[{value:"ES fields",id:"es-fields",children:[]},{value:"Developer fields",id:"developer-fields",children:[]},{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly)",id:"source-fields-destination-fields-similarly",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly)",id:"source-science-registry-fields--destination-fields-similarly",children:[]},{value:"&quot;Preferred&quot; fields",id:"preferred-fields",children:[]},{value:"Values",id:"values",children:[]},{value:"Tstat Values",id:"tstat-values",children:[]}]}],o={toc:c};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(l.b)("wrapper",Object(n.a)({},o,a,{components:t,mdxType:"MDXLayout"}),Object(l.b)("h1",{id:"logstash"},"Logstash"),Object(l.b)("p",null,"These Logstash config files are in /etc/logstash/conf.d/"),Object(l.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(l.b)("p",null,"The following steps are defined for logstash:"),Object(l.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(l.b)("p",null,'Reads flows from a rabbitmq queue. (".disabled" can be removed from other input configs to get flows from other sources.)'),Object(l.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(l.b)("p",null,"Drops flows to or from private IP addresses;\nadds @ingest_time (this is mainly for developers);\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions"),Object(l.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(l.b)("p",null,"Adds a unique id based on the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This ends up being called meta.id."),Object(l.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(l.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(l.b)("p",null,"Notes: By default, 5-minute nfcapd files are assumed, and if less than 10.5 min have passed between the start of the current flow and the start of the last matching one, stitch the two together."),Object(l.b)("p",null,"Your logstash pipeline can have only 1 worker or aggregation is not going to work! "),Object(l.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(l.b)("p",null,'If the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast";\nqueries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes. '),Object(l.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(l.b)("p",null,"Normally, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, try getting an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(l.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(l.b)("p",null,"Uses the ASN determined previously to get the organization name from the prepared CAIDA lookup file."),Object(l.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(l.b)("p",null,"Search (optional) lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(l.b)("p",null,"Notes: These lookup tables are not stored in github."),Object(l.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(l.b)("p",null,"Uses a fake geoip database containing Science Registry information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves scireg fields we don't need to save to elasticsearch."),Object(l.b)("p",null,"Notes: The science registry fake geoip database can be downloaded from scienceregistry.grnoc.iu.edu via wget in a cron job. "),Object(l.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(l.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them. "),Object(l.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(l.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(l.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(l.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the preferred_organization and preferred_location fields."),Object(l.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(l.b)("p",null,"Sets additional quick and easy fields.  Currently we have:\n",Object(l.b)("em",{parentName:"p"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)\n"),"sensor_type    = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)\n",Object(l.b)("em",{parentName:"p"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)\n"),"is_network_testing = yes, no  (yes if discipline = CS.Network Testing and Monitoring or port = 5001, 5101, or 5201)"),Object(l.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(l.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(l.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(l.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(l.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(l.b)("p",null,'Sends results to a rabbitmq queue (".disabled" can be removed from other output configs to send flows to other places)'),Object(l.b)("h3",{id:"final-stage"},"Final Stage"),Object(l.b)("p",null,"In our case, OmniSOC manages the last stage. Their logstash instance reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-",Object(l.b)("em",{parentName:"p"}," (or om-ns-ilight-"),", etc).  "),Object(l.b)("p",null,"This can be easily replicated with the following configuration though you'll need one for each feed/index."),Object(l.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(l.b)("pre",null,Object(l.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(l.b)("p",null,"Once the data is published in elastic, you can use the ",Object(l.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboard")," to visualize the data."),Object(l.b)("h2",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("h3",{id:"es-fields"},"ES fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06."),Object(l.b)("td",{parentName:"tr",align:null},"equivalent to an sql table")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"document id, set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))),Object(l.b)("h3",{id:"developer-fields"},"Developer fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"type"),Object(l.b)("td",{parentName:"tr",align:null},"flow"),Object(l.b)("td",{parentName:"tr",align:null},'Always "flow" for us. Other types may be "macy", etc.')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@injest_time"),Object(l.b)("td",{parentName:"tr",align:null},"2020-06-09T21:51:57.059Z"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline (99-outputs.conf)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @injest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.")))),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned flow id")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"Sflow, Netflow, or Tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor group")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor type")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly"},"Source Fields (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"ASN of the IP from geoip ASN database or the ASN from the flow header")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS of the IP from geoip ASN database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly"},"Source Science Registry Fields  (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford - ImageNet"),Object(l.b)("td",{parentName:"tr",align:null},"Resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"-"),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"CS. Intelligent Systems"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"."),Object(l.b)("td",{parentName:"tr",align:null},"Can be an array of projects ","[we may change this field name soon]")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's location, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"preferred-fields"},'"Preferred" fields'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Otherwise, they are the geoip values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"values"},"Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-values"},"Tstat Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))))}s.isMDXComponent=!0},228:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function b(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){l(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function c(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var o=r.a.createContext({}),s=function(e){var t=r.a.useContext(o),a=t;return e&&(a="function"==typeof e?e(t):b(b({},t),e)),a},d=function(e){var t=s(e.components);return r.a.createElement(o.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,o=c(e,["components","mdxType","originalType","parentName"]),d=s(a),m=n,u=d["".concat(i,".").concat(m)]||d[m]||p[m]||l;return a?r.a.createElement(u,b(b({ref:t},o),{},{components:a})):r.a.createElement(u,b({ref:t},o))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,i=new Array(l);i[0]=m;var b={};for(var c in t)hasOwnProperty.call(t,c)&&(b[c]=t[c]);b.originalType=e,b.mdxType="string"==typeof e?e:n,i[1]=b;for(var o=2;o<l;o++)i[o]=a[o];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/3914abfa.09b700f7.js b/assets/js/3914abfa.df468ae2.js
similarity index 96%
rename from assets/js/3914abfa.09b700f7.js
rename to assets/js/3914abfa.df468ae2.js
index bf2e70d4..a17cf45e 100644
--- a/assets/js/3914abfa.09b700f7.js
+++ b/assets/js/3914abfa.df468ae2.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[31],{101:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.7/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",version:"1.2.7",sidebar_label:"Replay Dataset",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," the best\ntest is to replay network data and inspecting the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been annonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files locally from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note on which port the collectors are listing on.  You'll be running the collectors by using a custom docker-compose.override.yml but if you're using the example provided as baseline the ports should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow."),Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow and ")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data use the following command for netflow/sflow respectively"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9998 -r sflow_anon.nfcapd -v 9 -d 1000\n")))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[32],{102:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.7/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",version:"1.2.7",sidebar_label:"Replay Dataset",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," the best\ntest is to replay network data and inspecting the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been annonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files locally from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note on which port the collectors are listing on.  You'll be running the collectors by using a custom docker-compose.override.yml but if you're using the example provided as baseline the ports should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow."),Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow and ")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data use the following command for netflow/sflow respectively"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9998 -r sflow_anon.nfcapd -v 9 -d 1000\n")))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/40b960a4.2c1c192b.js b/assets/js/40b960a4.422649da.js
similarity index 96%
rename from assets/js/40b960a4.2c1c192b.js
rename to assets/js/40b960a4.422649da.js
index 171fca98..59ba4db8 100644
--- a/assets/js/40b960a4.2c1c192b.js
+++ b/assets/js/40b960a4.422649da.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[32],{102:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.9/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",version:"1.2.9",sidebar_label:"Replay Dataset",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[33],{103:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.9/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",version:"1.2.9",sidebar_label:"Replay Dataset",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/41980e0e.34ea6f7b.js b/assets/js/41980e0e.d717c996.js
similarity index 98%
rename from assets/js/41980e0e.34ea6f7b.js
rename to assets/js/41980e0e.d717c996.js
index 2b2e395b..39c26a42 100644
--- a/assets/js/41980e0e.34ea6f7b.js
+++ b/assets/js/41980e0e.d717c996.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[34],{104:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),i=(n(0),n(232)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-1.2.11/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-1.2.11/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/docker.md",version:"1.2.11",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.11/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"build-images"},"Build Images"),Object(i.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(i.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(i.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(i.b)("inlineCode",{parentName:"p"}," "),"  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(i.b)("inlineCode",{parentName:"p"}," ")))),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(a,".").concat(u)]||p[u]||b[u]||i;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,a[1]=c;for(var s=2;s<i;s++)a[s]=n[s];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[34],{104:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),i=(n(0),n(228)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-1.2.11/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-1.2.11/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/docker.md",version:"1.2.11",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.11/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"build-images"},"Build Images"),Object(i.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(i.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(i.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(i.b)("inlineCode",{parentName:"p"}," "),"  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(i.b)("inlineCode",{parentName:"p"}," ")))),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(a,".").concat(u)]||p[u]||b[u]||i;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,a[1]=c;for(var s=2;s<i;s++)a[s]=n[s];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/42e094bc.2568d8ec.js b/assets/js/42e094bc.04cc4a66.js
similarity index 99%
rename from assets/js/42e094bc.2568d8ec.js
rename to assets/js/42e094bc.04cc4a66.js
index 8150310b..da6258f0 100644
--- a/assets/js/42e094bc.2568d8ec.js
+++ b/assets/js/42e094bc.04cc4a66.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[35],{105:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.8/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",version:"1.2.8",sidebar_label:"Docker Simple",sidebar:"version-1.2.8/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}]},{value:"Upgrading",id:"upgrading",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)"),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_name\n")),Object(r.b)("p",null,'Replace "tag_name" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h3",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("p",null,Object(r.b)("h3",{parentName:"p"},"Shut things down"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker_compose down\n")),Object(r.b)("p",{parentName:"p"},"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(r.b)("h3",{parentName:"p"},"Update Source Code"),Object(r.b)("p",{parentName:"p"},"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(r.b)("p",{parentName:"div"},"Example:\n",Object(r.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(r.b)("h3",{parentName:"p"},"Check/Update Files"),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(r.b)("h3",{parentName:"p"},"Select Release Version"),Object(r.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_value \n./scripts/docker_select_version.sh\n")),Object(r.b)("p",{parentName:"p"},"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(r.b)("h3",{parentName:"p"},"Update Docker Containers"),Object(r.b)("p",{parentName:"p"},"Do not forget this stop!  This applies for both development and release versions."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(r.b)("h3",{parentName:"p"},"Restart Docker Containers"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),m=a,h=d["".concat(i,".").concat(m)]||d[m]||b[m]||r;return n?o.a.createElement(h,l(l({ref:t},c),{},{components:n})):o.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[35],{105:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.8/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",version:"1.2.8",sidebar_label:"Docker Simple",sidebar:"version-1.2.8/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}]},{value:"Upgrading",id:"upgrading",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)"),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_name\n")),Object(r.b)("p",null,'Replace "tag_name" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h3",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("p",null,Object(r.b)("h3",{parentName:"p"},"Shut things down"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker_compose down\n")),Object(r.b)("p",{parentName:"p"},"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(r.b)("h3",{parentName:"p"},"Update Source Code"),Object(r.b)("p",{parentName:"p"},"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(r.b)("p",{parentName:"div"},"Example:\n",Object(r.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(r.b)("h3",{parentName:"p"},"Check/Update Files"),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(r.b)("h3",{parentName:"p"},"Select Release Version"),Object(r.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_value \n./scripts/docker_select_version.sh\n")),Object(r.b)("p",{parentName:"p"},"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(r.b)("h3",{parentName:"p"},"Update Docker Containers"),Object(r.b)("p",{parentName:"p"},"Do not forget this stop!  This applies for both development and release versions."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(r.b)("h3",{parentName:"p"},"Restart Docker Containers"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),m=a,h=d["".concat(i,".").concat(m)]||d[m]||b[m]||r;return n?o.a.createElement(h,l(l({ref:t},c),{},{components:n})):o.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/42f0e773.55a7a6c7.js b/assets/js/42f0e773.73889b7e.js
similarity index 97%
rename from assets/js/42f0e773.55a7a6c7.js
rename to assets/js/42f0e773.73889b7e.js
index 36b89e35..939ca307 100644
--- a/assets/js/42f0e773.55a7a6c7.js
+++ b/assets/js/42f0e773.73889b7e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[36],{106:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.8/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.8/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_pipeline.md",version:"1.2.8"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[36],{106:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.8/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.8/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_pipeline.md",version:"1.2.8"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/42f7ba93.8e66f0a3.js b/assets/js/42f7ba93.e851b609.js
similarity index 97%
rename from assets/js/42f7ba93.8e66f0a3.js
rename to assets/js/42f7ba93.e851b609.js
index b5bd05e6..1b6c66ee 100644
--- a/assets/js/42f7ba93.8e66f0a3.js
+++ b/assets/js/42f7ba93.e851b609.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[37],{107:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.7/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.7/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/nfdump.md",version:"1.2.7",sidebar_label:"Nfdump",sidebar:"version-1.2.7/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[37],{107:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.7/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.7/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/nfdump.md",version:"1.2.7",sidebar_label:"Nfdump",sidebar:"version-1.2.7/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/43862f47.792b1adb.js b/assets/js/43862f47.7a6ac57b.js
similarity index 97%
rename from assets/js/43862f47.792b1adb.js
rename to assets/js/43862f47.7a6ac57b.js
index 0b6d1985..c6bea67e 100644
--- a/assets/js/43862f47.792b1adb.js
+++ b/assets/js/43862f47.7a6ac57b.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[38],{108:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.7/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.7/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/importer.md",version:"1.2.7",sidebar_label:"Importer",sidebar:"version-1.2.7/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[38],{108:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.7/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.7/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/importer.md",version:"1.2.7",sidebar_label:"Importer",sidebar:"version-1.2.7/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/46a600e4.c1d369e6.js b/assets/js/46a600e4.790fa3f6.js
similarity index 97%
rename from assets/js/46a600e4.c1d369e6.js
rename to assets/js/46a600e4.790fa3f6.js
index 9758e454..ca0d18af 100644
--- a/assets/js/46a600e4.c1d369e6.js
+++ b/assets/js/46a600e4.790fa3f6.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[39],{109:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.10/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.10/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_pipeline.md",version:"1.2.10"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[39],{109:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.10/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.10/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_pipeline.md",version:"1.2.10"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/480c5911.4a2f1d5a.js b/assets/js/480c5911.48d5baca.js
similarity index 97%
rename from assets/js/480c5911.4a2f1d5a.js
rename to assets/js/480c5911.48d5baca.js
index 76b9e52d..0687289e 100644
--- a/assets/js/480c5911.4a2f1d5a.js
+++ b/assets/js/480c5911.48d5baca.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[40],{110:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return u})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.5/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.",source:"@site/versioned_docs/version-1.2.5/DOCUMENTATION.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.5/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/DOCUMENTATION.md",version:"1.2.5",sidebar_label:"Documentation Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}},u=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]}],l={toc:u};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  "),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h3",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h3",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h3",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function u(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),s=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=u(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,m=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var u in t)hasOwnProperty.call(t,u)&&(c[u]=t[u]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[40],{110:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return u})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.5/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.",source:"@site/versioned_docs/version-1.2.5/DOCUMENTATION.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.5/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/DOCUMENTATION.md",version:"1.2.5",sidebar_label:"Documentation Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}},u=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]}],l={toc:u};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  "),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h3",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h3",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h3",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function u(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),s=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=u(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,m=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var u in t)hasOwnProperty.call(t,u)&&(c[u]=t[u]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/49703371.0ed6dd81.js b/assets/js/49703371.0286d91d.js
similarity index 99%
rename from assets/js/49703371.0ed6dd81.js
rename to assets/js/49703371.0286d91d.js
index a60831bc..d83d3825 100644
--- a/assets/js/49703371.0ed6dd81.js
+++ b/assets/js/49703371.0286d91d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[41],{111:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.8/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",version:"1.2.8",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=o,h=u["".concat(l,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=b;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[41],{111:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(228)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.8/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",version:"1.2.8",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=o,h=u["".concat(l,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=b;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/49e74cdd.e8f8d5a1.js b/assets/js/49e74cdd.00e19770.js
similarity index 98%
rename from assets/js/49e74cdd.e8f8d5a1.js
rename to assets/js/49e74cdd.00e19770.js
index d713e493..1df320c5 100644
--- a/assets/js/49e74cdd.e8f8d5a1.js
+++ b/assets/js/49e74cdd.00e19770.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[42],{112:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(232)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.10/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.10/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/intro.md",version:"1.2.10",sidebar_label:"Intro",sidebar:"version-1.2.10/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[42],{112:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(228)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.10/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.10/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/intro.md",version:"1.2.10",sidebar_label:"Intro",sidebar:"version-1.2.10/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/4a74d8e8.964ee9cc.js b/assets/js/4a74d8e8.d930cabd.js
similarity index 99%
rename from assets/js/4a74d8e8.964ee9cc.js
rename to assets/js/4a74d8e8.d930cabd.js
index 50946f69..e168b997 100644
--- a/assets/js/4a74d8e8.964ee9cc.js
+++ b/assets/js/4a74d8e8.d930cabd.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[43],{113:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.10/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.10/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/elastic_search.md",version:"1.2.10",sidebar_label:"Elasticsearch",sidebar:"version-1.2.10/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[43],{113:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.10/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.10/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/elastic_search.md",version:"1.2.10",sidebar_label:"Elasticsearch",sidebar:"version-1.2.10/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/4ff24bf8.5f1f44d6.js b/assets/js/4ff24bf8.c0099967.js
similarity index 97%
rename from assets/js/4ff24bf8.5f1f44d6.js
rename to assets/js/4ff24bf8.c0099967.js
index f9e95c55..e3c00cf1 100644
--- a/assets/js/4ff24bf8.5f1f44d6.js
+++ b/assets/js/4ff24bf8.c0099967.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[44],{114:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return i})),r.d(t,"toc",(function(){return l})),r.d(t,"default",(function(){return p}));var n=r(3),o=r(7),c=(r(0),r(232)),a={},i={unversionedId:"components/docker_upgrade",id:"version-1.2.7/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Update Source Code",source:"@site/versioned_docs/version-1.2.7/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_upgrade.md",version:"1.2.7"},l=[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Docker and Collectors",id:"docker-and-collectors",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}],s={toc:l};function p(e){var t=e.components,r=Object(o.a)(e,["components"]);return Object(c.b)("wrapper",Object(n.a)({},s,r,{components:t,mdxType:"MDXLayout"}),Object(c.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(c.b)("p",null,"To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  "),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(c.b)("h3",{id:"docker-and-collectors"},"Docker and Collectors"),Object(c.b)("p",null,"Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, ",Object(c.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),"). Also check the docker version (eg, ",Object(c.b)("inlineCode",{parentName:"p"},'version: "3.7"'),") to see if you'll need to ugrade docker."),Object(c.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(c.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},'git checkout "tag_value" \n./scripts/docker_select_version.sh\n')),Object(c.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(c.b)("p",null,"This applies for both development and release"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre"},"docker-compose pull\n")))}p.isMDXComponent=!0},232:function(e,t,r){"use strict";r.d(t,"a",(function(){return d})),r.d(t,"b",(function(){return m}));var n=r(0),o=r.n(n);function c(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){c(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function l(e,t){if(null==e)return{};var r,n,o=function(e,t){if(null==e)return{};var r,n,o={},c=Object.keys(e);for(n=0;n<c.length;n++)r=c[n],t.indexOf(r)>=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n<c.length;n++)r=c[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var s=o.a.createContext({}),p=function(e){var t=o.a.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},d=function(e){var t=p(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,c=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),d=p(r),b=n,m=d["".concat(a,".").concat(b)]||d[b]||u[b]||c;return r?o.a.createElement(m,i(i({ref:t},s),{},{components:r})):o.a.createElement(m,i({ref:t},s))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var c=r.length,a=new Array(c);a[0]=b;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:n,a[1]=i;for(var s=2;s<c;s++)a[s]=r[s];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,r)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[44],{114:function(e,t,r){"use strict";r.r(t),r.d(t,"frontMatter",(function(){return a})),r.d(t,"metadata",(function(){return i})),r.d(t,"toc",(function(){return l})),r.d(t,"default",(function(){return p}));var n=r(3),o=r(7),c=(r(0),r(228)),a={},i={unversionedId:"components/docker_upgrade",id:"version-1.2.7/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Update Source Code",source:"@site/versioned_docs/version-1.2.7/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_upgrade.md",version:"1.2.7"},l=[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Docker and Collectors",id:"docker-and-collectors",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}],s={toc:l};function p(e){var t=e.components,r=Object(o.a)(e,["components"]);return Object(c.b)("wrapper",Object(n.a)({},s,r,{components:t,mdxType:"MDXLayout"}),Object(c.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(c.b)("p",null,"To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  "),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(c.b)("h3",{id:"docker-and-collectors"},"Docker and Collectors"),Object(c.b)("p",null,"Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, ",Object(c.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),"). Also check the docker version (eg, ",Object(c.b)("inlineCode",{parentName:"p"},'version: "3.7"'),") to see if you'll need to ugrade docker."),Object(c.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(c.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},'git checkout "tag_value" \n./scripts/docker_select_version.sh\n')),Object(c.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(c.b)("p",null,"This applies for both development and release"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre"},"docker-compose pull\n")))}p.isMDXComponent=!0},228:function(e,t,r){"use strict";r.d(t,"a",(function(){return d})),r.d(t,"b",(function(){return m}));var n=r(0),o=r.n(n);function c(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){c(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function l(e,t){if(null==e)return{};var r,n,o=function(e,t){if(null==e)return{};var r,n,o={},c=Object.keys(e);for(n=0;n<c.length;n++)r=c[n],t.indexOf(r)>=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n<c.length;n++)r=c[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var s=o.a.createContext({}),p=function(e){var t=o.a.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},d=function(e){var t=p(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,c=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),d=p(r),b=n,m=d["".concat(a,".").concat(b)]||d[b]||u[b]||c;return r?o.a.createElement(m,i(i({ref:t},s),{},{components:r})):o.a.createElement(m,i({ref:t},s))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var c=r.length,a=new Array(c);a[0]=b;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:n,a[1]=i;for(var s=2;s<c;s++)a[s]=r[s];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,r)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/4ffdf56a.031ac316.js b/assets/js/4ffdf56a.f556f662.js
similarity index 98%
rename from assets/js/4ffdf56a.031ac316.js
rename to assets/js/4ffdf56a.f556f662.js
index 7d232eb7..994b3f80 100644
--- a/assets/js/4ffdf56a.031ac316.js
+++ b/assets/js/4ffdf56a.f556f662.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[45],{115:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(232)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/docs/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/next/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/documentation_guide.md",version:"current",sidebar_label:"Docusaurus",sidebar:"Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json"),Object(i.b)("li",{parentName:"ul"},"change 1.2.6 in docusaurus.config.js back to 1.2.5")),Object(i.b)("p",null,"Then git add, commit, and push"),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[45],{115:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(228)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/docs/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/next/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/documentation_guide.md",version:"current",sidebar_label:"Docusaurus",sidebar:"Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json"),Object(i.b)("li",{parentName:"ul"},"change 1.2.6 in docusaurus.config.js back to 1.2.5")),Object(i.b)("p",null,"Then git add, commit, and push"),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/50032990.0ea61ccd.js b/assets/js/50032990.47d0b7f0.js
similarity index 97%
rename from assets/js/50032990.0ea61ccd.js
rename to assets/js/50032990.47d0b7f0.js
index 7a85cf3d..2259fe05 100644
--- a/assets/js/50032990.0ea61ccd.js
+++ b/assets/js/50032990.47d0b7f0.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[46],{116:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.9/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.9/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/tstat.md",version:"1.2.9",sidebar_label:"Tstat",sidebar:"version-1.2.9/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[46],{116:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.9/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.9/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/tstat.md",version:"1.2.9",sidebar_label:"Tstat",sidebar:"version-1.2.9/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/514d766a.b0ec51ab.js b/assets/js/514d766a.ca35f79e.js
similarity index 98%
rename from assets/js/514d766a.b0ec51ab.js
rename to assets/js/514d766a.ca35f79e.js
index 8dbbf4f5..04ae9081 100644
--- a/assets/js/514d766a.b0ec51ab.js
+++ b/assets/js/514d766a.ca35f79e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[47],{117:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(232)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-1.2.10/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-1.2.10/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/documentation_guide.md",version:"1.2.10",sidebar_label:"Docusaurus",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"},next:{title:"How to Tag a New Release",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"local-development"},"Local Development"),Object(i.b)("p",null,"If you are working on your local machine, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 for example."),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[47],{117:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(228)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-1.2.10/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-1.2.10/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/documentation_guide.md",version:"1.2.10",sidebar_label:"Docusaurus",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"},next:{title:"How to Tag a New Release",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"local-development"},"Local Development"),Object(i.b)("p",null,"If you are working on your local machine, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 for example."),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/53b0b5d8.144adf6f.js b/assets/js/53b0b5d8.144adf6f.js
deleted file mode 100644
index 43ba69ef..00000000
--- a/assets/js/53b0b5d8.144adf6f.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[48],{118:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),i=(n(0),n(232)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-2.0.0/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-2.0.0/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/docker.md",version:"2.0.0",sidebar_label:"Docker Dev Guide",sidebar:"version-2.0.0/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply skip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"build-images"},"Build Images"),Object(i.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(i.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(i.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(i.b)("inlineCode",{parentName:"p"}," "),"  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(i.b)("inlineCode",{parentName:"p"}," ")))),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(a,".").concat(u)]||p[u]||b[u]||i;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,a[1]=c;for(var s=2;s<i;s++)a[s]=n[s];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/56776a50.28321f6f.js b/assets/js/56776a50.009ef339.js
similarity index 98%
rename from assets/js/56776a50.28321f6f.js
rename to assets/js/56776a50.009ef339.js
index f995fdb6..5cd6d8f8 100644
--- a/assets/js/56776a50.28321f6f.js
+++ b/assets/js/56776a50.009ef339.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[49],{119:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return l})),o.d(t,"metadata",(function(){return i})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return d}));var a=o(3),n=o(7),r=(o(0),o(232)),l={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},i={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.8/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",version:"1.2.8",sidebar_label:"Docker Advanced",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. Edit docker-compose.override.yml",id:"1-edit-docker-composeoverrideyml",children:[]},{value:"2.  Edit netsage_override.xml",id:"2--edit-netsage_overridexml",children:[]},{value:"3. Edit environment file",id:"3-edit-environment-file",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"To Change the Culling of Nfcapd Files",id:"to-change-the-culling-of-nfcapd-files",children:[]},{value:"For Tstat Data",id:"for-tstat-data",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"For Data Saved to an NFS Volume",id:"for-data-saved-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,o=Object(n.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Simple installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-edit-docker-composeoverrideyml"},"1. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--edit-netsage_overridexml"},"2.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-edit-environment-file"},"3. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-change-the-culling-of-nfcapd-files"},"To Change the Culling of Nfcapd Files"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"for-tstat-data"},"For Tstat Data"),Object(r.b)("p",null,'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",null,"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),"."),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",null,"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))))}d.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return h}));var a=o(0),n=o.n(a);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function i(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?l(Object(o),!0).forEach((function(t){r(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):l(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,a,n=function(e,t){if(null==e)return{};var o,a,n={},r=Object.keys(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var c=n.a.createContext({}),d=function(e){var t=n.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},p=function(e){var t=d(e.components);return n.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.a.createElement(n.a.Fragment,{},t)}},m=n.a.forwardRef((function(e,t){var o=e.components,a=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(o),m=a,h=p["".concat(l,".").concat(m)]||p[m]||u[m]||r;return o?n.a.createElement(h,i(i({ref:t},c),{},{components:o})):n.a.createElement(h,i({ref:t},c))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=o.length,l=new Array(r);l[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:a,l[1]=i;for(var c=2;c<r;c++)l[c]=o[c];return n.a.createElement.apply(null,l)}return n.a.createElement.apply(null,o)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[48],{118:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return l})),o.d(t,"metadata",(function(){return i})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return d}));var a=o(3),n=o(7),r=(o(0),o(228)),l={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},i={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.8/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",version:"1.2.8",sidebar_label:"Docker Advanced",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. Edit docker-compose.override.yml",id:"1-edit-docker-composeoverrideyml",children:[]},{value:"2.  Edit netsage_override.xml",id:"2--edit-netsage_overridexml",children:[]},{value:"3. Edit environment file",id:"3-edit-environment-file",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"To Change the Culling of Nfcapd Files",id:"to-change-the-culling-of-nfcapd-files",children:[]},{value:"For Tstat Data",id:"for-tstat-data",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"For Data Saved to an NFS Volume",id:"for-data-saved-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,o=Object(n.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Simple installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-edit-docker-composeoverrideyml"},"1. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--edit-netsage_overridexml"},"2.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-edit-environment-file"},"3. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-change-the-culling-of-nfcapd-files"},"To Change the Culling of Nfcapd Files"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"for-tstat-data"},"For Tstat Data"),Object(r.b)("p",null,'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",null,"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),"."),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",null,"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))))}d.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return h}));var a=o(0),n=o.n(a);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function i(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?l(Object(o),!0).forEach((function(t){r(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):l(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,a,n=function(e,t){if(null==e)return{};var o,a,n={},r=Object.keys(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var c=n.a.createContext({}),d=function(e){var t=n.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},p=function(e){var t=d(e.components);return n.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.a.createElement(n.a.Fragment,{},t)}},m=n.a.forwardRef((function(e,t){var o=e.components,a=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(o),m=a,h=p["".concat(l,".").concat(m)]||p[m]||u[m]||r;return o?n.a.createElement(h,i(i({ref:t},c),{},{components:o})):n.a.createElement(h,i({ref:t},c))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=o.length,l=new Array(r);l[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:a,l[1]=i;for(var c=2;c<r;c++)l[c]=o[c];return n.a.createElement.apply(null,l)}return n.a.createElement.apply(null,o)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/56a29d1a.92eaa661.js b/assets/js/56a29d1a.de0b8997.js
similarity index 99%
rename from assets/js/56a29d1a.92eaa661.js
rename to assets/js/56a29d1a.de0b8997.js
index fd78c4f8..5254b05b 100644
--- a/assets/js/56a29d1a.92eaa661.js
+++ b/assets/js/56a29d1a.de0b8997.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[50],{120:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.9/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",version:"1.2.9",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files. "),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?a.a.createElement(b,r(r({ref:t},c),{},{components:n})):a.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[49],{119:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(228)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.9/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",version:"1.2.9",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files. "),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?a.a.createElement(b,r(r({ref:t},c),{},{components:n})):a.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/58d958f4.6a5669a4.js b/assets/js/58d958f4.572280db.js
similarity index 97%
rename from assets/js/58d958f4.6a5669a4.js
rename to assets/js/58d958f4.572280db.js
index a9f52641..752060d1 100644
--- a/assets/js/58d958f4.6a5669a4.js
+++ b/assets/js/58d958f4.572280db.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[51],{121:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.8/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.8/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.8/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/documentation_guide.md",version:"1.2.8",sidebar_label:"Documentation Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[50],{120:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.8/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.8/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.8/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/documentation_guide.md",version:"1.2.8",sidebar_label:"Documentation Guide",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/59305d05.7e6bc501.js b/assets/js/59305d05.7e6bc501.js
deleted file mode 100644
index 631b2fe9..00000000
--- a/assets/js/59305d05.7e6bc501.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[52],{122:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-2.0.0/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-2.0.0/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/nfdump.md",version:"2.0.0",sidebar_label:"Sflow/Netflow Data",sidebar:"version-2.0.0/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/5a20f281.847d72f0.js b/assets/js/5a20f281.84ff2815.js
similarity index 99%
rename from assets/js/5a20f281.847d72f0.js
rename to assets/js/5a20f281.84ff2815.js
index 5e9bfc63..ab72f067 100644
--- a/assets/js/5a20f281.847d72f0.js
+++ b/assets/js/5a20f281.84ff2815.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[53],{123:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.9/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.9/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/elastic_search.md",version:"1.2.9",sidebar_label:"Elasticsearch",sidebar:"version-1.2.9/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[51],{121:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.9/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.9/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/elastic_search.md",version:"1.2.9",sidebar_label:"Elasticsearch",sidebar:"version-1.2.9/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/5b6c4cb7.5d8d3771.js b/assets/js/5b6c4cb7.1e5700c2.js
similarity index 99%
rename from assets/js/5b6c4cb7.5d8d3771.js
rename to assets/js/5b6c4cb7.1e5700c2.js
index 2a499a5d..588d0926 100644
--- a/assets/js/5b6c4cb7.5d8d3771.js
+++ b/assets/js/5b6c4cb7.1e5700c2.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[54],{124:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.11/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.11/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/elastic_search.md",version:"1.2.11",sidebar_label:"Elasticsearch",sidebar:"version-1.2.11/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[52],{122:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.11/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.11/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/elastic_search.md",version:"1.2.11",sidebar_label:"Elasticsearch",sidebar:"version-1.2.11/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.11/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/5c4beb71.a62d61d9.js b/assets/js/5c4beb71.b666c6b3.js
similarity index 96%
rename from assets/js/5c4beb71.a62d61d9.js
rename to assets/js/5c4beb71.b666c6b3.js
index 7175f150..1d1860ca 100644
--- a/assets/js/5c4beb71.a62d61d9.js
+++ b/assets/js/5c4beb71.b666c6b3.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[55],{125:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(232)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.7/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.7/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/choosing.md",version:"1.2.7",sidebar_label:"Choose Install",sidebar:"version-1.2.7/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[53],{123:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.7/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.7/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/choosing.md",version:"1.2.7",sidebar_label:"Choose Install",sidebar:"version-1.2.7/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/5ee17acf.eeb64f93.js b/assets/js/5ee17acf.4e665490.js
similarity index 99%
rename from assets/js/5ee17acf.eeb64f93.js
rename to assets/js/5ee17acf.4e665490.js
index 2376ee09..5b65446f 100644
--- a/assets/js/5ee17acf.eeb64f93.js
+++ b/assets/js/5ee17acf.4e665490.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[56],{126:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return s})),a.d(t,"toc",(function(){return l})),a.d(t,"default",(function(){return d}));var n=a(3),o=a(7),r=(a(0),a(232)),i={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},s={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.9/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",version:"1.2.9",sidebar_label:"Docker Advanced",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"}},l=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. Edit docker-compose.override.yml",id:"1-edit-docker-composeoverrideyml",children:[]},{value:"2.  Edit netsage_override.xml",id:"2--edit-netsage_overridexml",children:[]},{value:"3. Edit environment file",id:"3-edit-environment-file",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]}],c={toc:l};function d(e){var t=e.components,a=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(n.a)({},c,a,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Simple installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-edit-docker-composeoverrideyml"},"1. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--edit-netsage_overridexml"},"2.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-edit-environment-file"},"3. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id) changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."),Object(r.b)("h2",{parentName:"div",id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",{parentName:"div"},"When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",{parentName:"div"},'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,'),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{parentName:"div",id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",{parentName:"div"},"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",{parentName:"div"},"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",{parentName:"div"},"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{parentName:"div",id:"to-process-tstat-data"},"To Process Tstat Data"),Object(r.b)("p",{parentName:"div"},'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",{parentName:"div"},"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),". Basically, you need to have Tstat send data directly to the same rabbit queue that the importers write sflow and netflow data to and that the logstash pipeline reads from."),Object(r.b)("h2",{parentName:"div",id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",{parentName:"div"},"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",{parentName:"div"},"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("p",{parentName:"div"},"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",{parentName:"div"},Object(r.b)("li",{parentName:"ul"},"The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use."),Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.")),Object(r.b)("h2",{parentName:"div",id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",{parentName:"div"},"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",{parentName:"div"},"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{parentName:"div",id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",{parentName:"div"},"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",{parentName:"div"},Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",{parentName:"div"},"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."))),Object(r.b)("p",null,"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.\n:::"))}d.isMDXComponent=!0},232:function(e,t,a){"use strict";a.d(t,"a",(function(){return p})),a.d(t,"b",(function(){return u}));var n=a(0),o=a.n(n);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function s(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function l(e,t){if(null==e)return{};var a,n,o=function(e,t){if(null==e)return{};var a,n,o={},r=Object.keys(e);for(n=0;n<r.length;n++)a=r[n],t.indexOf(a)>=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n<r.length;n++)a=r[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var c=o.a.createContext({}),d=function(e){var t=o.a.useContext(c),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},p=function(e){var t=d(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,r=e.originalType,i=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=d(a),h=n,u=p["".concat(i,".").concat(h)]||p[h]||m[h]||r;return a?o.a.createElement(u,s(s({ref:t},c),{},{components:a})):o.a.createElement(u,s({ref:t},c))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var r=a.length,i=new Array(r);i[0]=h;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:n,i[1]=s;for(var c=2;c<r;c++)i[c]=a[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,a)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[54],{124:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return s})),a.d(t,"toc",(function(){return l})),a.d(t,"default",(function(){return d}));var n=a(3),o=a(7),r=(a(0),a(228)),i={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},s={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.9/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",version:"1.2.9",sidebar_label:"Docker Advanced",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"}},l=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. Edit docker-compose.override.yml",id:"1-edit-docker-composeoverrideyml",children:[]},{value:"2.  Edit netsage_override.xml",id:"2--edit-netsage_overridexml",children:[]},{value:"3. Edit environment file",id:"3-edit-environment-file",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]}],c={toc:l};function d(e){var t=e.components,a=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(n.a)({},c,a,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Simple installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-edit-docker-composeoverrideyml"},"1. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--edit-netsage_overridexml"},"2.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-edit-environment-file"},"3. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id) changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."),Object(r.b)("h2",{parentName:"div",id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",{parentName:"div"},"When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",{parentName:"div"},'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,'),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{parentName:"div",id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",{parentName:"div"},"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",{parentName:"div"},"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",{parentName:"div"},"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{parentName:"div",id:"to-process-tstat-data"},"To Process Tstat Data"),Object(r.b)("p",{parentName:"div"},'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",{parentName:"div"},"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),". Basically, you need to have Tstat send data directly to the same rabbit queue that the importers write sflow and netflow data to and that the logstash pipeline reads from."),Object(r.b)("h2",{parentName:"div",id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",{parentName:"div"},"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",{parentName:"div"},"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",{parentName:"div"},Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("p",{parentName:"div"},"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",{parentName:"div"},Object(r.b)("li",{parentName:"ul"},"The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use."),Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.")),Object(r.b)("h2",{parentName:"div",id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",{parentName:"div"},"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",{parentName:"div"},"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{parentName:"div",id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",{parentName:"div"},"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",{parentName:"div"},Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",{parentName:"div"},"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."))),Object(r.b)("p",null,"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.\n:::"))}d.isMDXComponent=!0},228:function(e,t,a){"use strict";a.d(t,"a",(function(){return p})),a.d(t,"b",(function(){return u}));var n=a(0),o=a.n(n);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function s(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function l(e,t){if(null==e)return{};var a,n,o=function(e,t){if(null==e)return{};var a,n,o={},r=Object.keys(e);for(n=0;n<r.length;n++)a=r[n],t.indexOf(a)>=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n<r.length;n++)a=r[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var c=o.a.createContext({}),d=function(e){var t=o.a.useContext(c),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},p=function(e){var t=d(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,r=e.originalType,i=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=d(a),h=n,u=p["".concat(i,".").concat(h)]||p[h]||m[h]||r;return a?o.a.createElement(u,s(s({ref:t},c),{},{components:a})):o.a.createElement(u,s({ref:t},c))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var r=a.length,i=new Array(r);i[0]=h;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:n,i[1]=s;for(var c=2;c<r;c++)i[c]=a[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,a)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/5ef24f1d.6c97f268.js b/assets/js/5ef24f1d.6c97f268.js
new file mode 100644
index 00000000..7a38e42c
--- /dev/null
+++ b/assets/js/5ef24f1d.6c97f268.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[55],{125:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-2.1.0/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-2.1.0/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/importer.md",version:"2.1.0",sidebar_label:"Importer",sidebar:"version-2.1.0/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/6125e2e5.da94766a.js b/assets/js/6125e2e5.372c6163.js
similarity index 97%
rename from assets/js/6125e2e5.da94766a.js
rename to assets/js/6125e2e5.372c6163.js
index b335ee2e..343c848a 100644
--- a/assets/js/6125e2e5.da94766a.js
+++ b/assets/js/6125e2e5.372c6163.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[57],{127:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.9/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.9/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_env.md",version:"1.2.9"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[56],{126:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.9/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.9/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_env.md",version:"1.2.9"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/63503174.24f9a6f6.js b/assets/js/63503174.24f9a6f6.js
new file mode 100644
index 00000000..6c8382a4
--- /dev/null
+++ b/assets/js/63503174.24f9a6f6.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[57],{127:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"2.1.0","label":"2.1.0","isLast":true,"docsSidebars":{"version-2.1.0/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/deploy/docker_install_advanced":"version-2.1.0/Pipeline","/netsage-pipeline/docs/deploy/docker_install_simple":"version-2.1.0/Pipeline","/netsage-pipeline/docs/deploy/docker_troubleshoot":"version-2.1.0/Pipeline","/netsage-pipeline/docs/deploy/docker_upgrade":"version-2.1.0/Pipeline","/netsage-pipeline/docs/devel/dev_dataset":"version-2.1.0/Pipeline","/netsage-pipeline/docs/devel/docker_dev_guide":"version-2.1.0/Pipeline","/netsage-pipeline/docs/devel/docker_dev_tag":"version-2.1.0/Pipeline","/netsage-pipeline/docs/devel/docusaurus":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/elastic":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/importer":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/intro":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/logstash":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/nfdump":"version-2.1.0/Pipeline","/netsage-pipeline/docs/pipeline/tstat":"version-2.1.0/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/63f5fe43.bed1e40c.js b/assets/js/63f5fe43.faa7d00d.js
similarity index 98%
rename from assets/js/63f5fe43.bed1e40c.js
rename to assets/js/63f5fe43.faa7d00d.js
index 2a1e0c10..586475d7 100644
--- a/assets/js/63f5fe43.bed1e40c.js
+++ b/assets/js/63f5fe43.faa7d00d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[58],{128:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),i=(n(0),n(232)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-1.2.10/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-1.2.10/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/docker.md",version:"1.2.10",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.10/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply scip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"build-images"},"Build Images"),Object(i.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(i.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(i.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(i.b)("inlineCode",{parentName:"p"}," "),"  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(i.b)("inlineCode",{parentName:"p"}," ")))),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(a,".").concat(u)]||p[u]||b[u]||i;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,a[1]=c;for(var s=2;s<i;s++)a[s]=n[s];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[58],{128:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),i=(n(0),n(228)),a={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},c={unversionedId:"devel/docker_dev_guide",id:"version-1.2.10/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Selecting a Version",source:"@site/versioned_docs/version-1.2.10/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/docker.md",version:"1.2.10",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.10/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"},next:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus"}},l=[{value:"Selecting a Version",id:"selecting-a-version",children:[]},{value:"Installing",id:"installing",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Build Images",id:"build-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]},{value:"Handy Docker Commands",id:"handy-docker-commands",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,'You can use the "master" version or a tagged version.',Object(i.b)("br",{parentName:"p"}),"\n","To select a released version use the docker_select_version.sh script (see the Deployment Guide).\nIf you wish to use the development version (master branch) simply scip the docker_select_version.sh step."),Object(i.b)("h2",{id:"installing"},"Installing"),Object(i.b)("p",null,"See the Deployment Guide to learn how to set up collectors, your environment and override files, etc."),Object(i.b)("h2",{id:"importer"},"Importer"),Object(i.b)("p",null,'The importer "shared" config that Docker uses is defined in compose/netsage_shared.xml.  ',Object(i.b)("strong",{parentName:"p"}," NOTE: If you want to make changes to this file, you will need to rebuild the container")),Object(i.b)("h2",{id:"build-images"},"Build Images"),Object(i.b)("p",null,"The images are published on Docker Hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in the GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(i.b)("p",null,"NOTE: The importer container includes the config files for the logstash pipeline. "),Object(i.b)("h2",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(i.b)("inlineCode",{parentName:"p"}," "),"  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(i.b)("inlineCode",{parentName:"p"}," ")))),Object(i.b)("h2",{id:"handy-docker-commands"},"Handy Docker Commands"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),d=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},u=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,m=p["".concat(a,".").concat(u)]||p[u]||b[u]||i;return n?r.a.createElement(m,c(c({ref:t},s),{},{components:n})):r.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:o,a[1]=c;for(var s=2;s<i;s++)a[s]=n[s];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/64269ed8.20920114.js b/assets/js/64269ed8.18f9818d.js
similarity index 98%
rename from assets/js/64269ed8.20920114.js
rename to assets/js/64269ed8.18f9818d.js
index bad7244a..4f6ba587 100644
--- a/assets/js/64269ed8.20920114.js
+++ b/assets/js/64269ed8.18f9818d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[59],{129:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={},c={unversionedId:"components/docker_env",id:"components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/docs/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/next/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_env.md",version:"current"},s=[],l={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,c(c({ref:t},l),{},{components:n})):a.a.createElement(d,c({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:o,i[1]=c;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[59],{129:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={},c={unversionedId:"components/docker_env",id:"components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/docs/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/next/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_env.md",version:"current"},s=[],l={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,c(c({ref:t},l),{},{components:n})):a.a.createElement(d,c({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:o,i[1]=c;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/653602f1.c1abe2fd.js b/assets/js/653602f1.d686a47d.js
similarity index 98%
rename from assets/js/653602f1.c1abe2fd.js
rename to assets/js/653602f1.d686a47d.js
index 841f1775..645c7de4 100644
--- a/assets/js/653602f1.c1abe2fd.js
+++ b/assets/js/653602f1.d686a47d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[60],{130:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/docs/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_troubleshooting.md",version:"current",sidebar_label:"Troubleshooting",sidebar:"Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(a.b)("inlineCode",{parentName:"li"},"docker ps")),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(a.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(a.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(a.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(a.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),b=o,f=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?r.a.createElement(f,l(l({ref:t},c),{},{components:n})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[60],{130:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/docs/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_troubleshooting.md",version:"current",sidebar_label:"Troubleshooting",sidebar:"Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(a.b)("inlineCode",{parentName:"li"},"docker ps")),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(a.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(a.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(a.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(a.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),b=o,f=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?r.a.createElement(f,l(l({ref:t},c),{},{components:n})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/6546ef57.e13c6102.js b/assets/js/6546ef57.b480f0cc.js
similarity index 98%
rename from assets/js/6546ef57.e13c6102.js
rename to assets/js/6546ef57.b480f0cc.js
index 876201c3..50b29a50 100644
--- a/assets/js/6546ef57.e13c6102.js
+++ b/assets/js/6546ef57.b480f0cc.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[61],{131:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_upgrade",id:"version-1.2.9/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Shut things down",source:"@site/versioned_docs/version-1.2.9/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_upgrade.md",version:"1.2.9"},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Files",id:"checkupdate-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart Docker Containers",id:"restart-docker-containers",children:[]},{value:"Delete old images",id:"delete-old-images",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-files"},"Check/Update Files"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b tag_value \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this step!  This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-docker-containers"},"Restart Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images"},"Delete old images"),Object(a.b)("p",null,"To save space, delete any old images that are not needed. "))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),m=r,b=p["".concat(i,".").concat(m)]||p[m]||u[m]||a;return n?o.a.createElement(b,c(c({ref:t},s),{},{components:n})):o.a.createElement(b,c({ref:t},s))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[61],{131:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_upgrade",id:"version-1.2.9/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Shut things down",source:"@site/versioned_docs/version-1.2.9/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_upgrade.md",version:"1.2.9"},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Files",id:"checkupdate-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart Docker Containers",id:"restart-docker-containers",children:[]},{value:"Delete old images",id:"delete-old-images",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-files"},"Check/Update Files"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b tag_value \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this step!  This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-docker-containers"},"Restart Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images"},"Delete old images"),Object(a.b)("p",null,"To save space, delete any old images that are not needed. "))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),m=r,b=p["".concat(i,".").concat(m)]||p[m]||u[m]||a;return n?o.a.createElement(b,c(c({ref:t},s),{},{components:n})):o.a.createElement(b,c({ref:t},s))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/66e31e6e.ce62be57.js b/assets/js/66e31e6e.ce62be57.js
deleted file mode 100644
index e0c1cbc0..00000000
--- a/assets/js/66e31e6e.ce62be57.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[62],{132:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/docs/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/next/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/choosing.md",version:"current",sidebar_label:"Choose Install",sidebar:"Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/next/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/next/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/69c1008e.e9443d0d.js b/assets/js/69c1008e.6660742f.js
similarity index 95%
rename from assets/js/69c1008e.e9443d0d.js
rename to assets/js/69c1008e.6660742f.js
index 442f85d0..6285161f 100644
--- a/assets/js/69c1008e.e9443d0d.js
+++ b/assets/js/69c1008e.6660742f.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[63],{133:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.11/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.11/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_pipeline.md",version:"1.2.11"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}l.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[62],{132:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.11/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.11/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_pipeline.md",version:"1.2.11"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}l.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/6a059f5b.6c7d9748.js b/assets/js/6a059f5b.57b36ba4.js
similarity index 98%
rename from assets/js/6a059f5b.6c7d9748.js
rename to assets/js/6a059f5b.57b36ba4.js
index 5cca4c7b..9dfe157f 100644
--- a/assets/js/6a059f5b.6c7d9748.js
+++ b/assets/js/6a059f5b.57b36ba4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[64],{134:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),o=n(7),a=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.8/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.8/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/logstash.md",version:"1.2.8",sidebar_label:"Logstash",sidebar:"version-1.2.8/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(a.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(a.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(a.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(a.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(a.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(a.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(a.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(a.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(a.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(a.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(a.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(a.b)("p",null,"Makes any changes to fields needed for specific sensors. It currently contains an adjustment to the number of bytes and packets for a pacificwave sensor with a complicated setup, and a change in sensor name for certain flows from the NEAAR sensor.  "),Object(a.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(a.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(a.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(a.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(a.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(a.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(a.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(a.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(a.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(a.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(a.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(a.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(a.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(a.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(a.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(a.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(a.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(a.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(a.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(a.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout."),Object(a.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(a.b)("p",null,"Uses a fake geoip database containing ",Object(a.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"The ",Object(a.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(a.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(a.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(a.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(a.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(a.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(a.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(a.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(a.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(a.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(a.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(a.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(a.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(a.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(a.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(a.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(a.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(a.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(a.b)("h3",{id:"final-stage"},"Final Stage"),Object(a.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(a.b)("h2",{id:"field-names"},"Field names"),Object(a.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(a.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var i=n(0),o=n.n(i);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,o=function(e,t){if(null==e)return{};var n,i,o={},a=Object.keys(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,a=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),u=d(n),p=i,h=u["".concat(s,".").concat(p)]||u[p]||f[p]||a;return n?o.a.createElement(h,r(r({ref:t},l),{},{components:n})):o.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=n.length,s=new Array(a);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<a;l++)s[l]=n[l];return o.a.createElement.apply(null,s)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[63],{133:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),o=n(7),a=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.8/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.8/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/logstash.md",version:"1.2.8",sidebar_label:"Logstash",sidebar:"version-1.2.8/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(a.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(a.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(a.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(a.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(a.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(a.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(a.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(a.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(a.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(a.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(a.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(a.b)("p",null,"Makes any changes to fields needed for specific sensors. It currently contains an adjustment to the number of bytes and packets for a pacificwave sensor with a complicated setup, and a change in sensor name for certain flows from the NEAAR sensor.  "),Object(a.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(a.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(a.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(a.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(a.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(a.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(a.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(a.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(a.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(a.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(a.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(a.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(a.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(a.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(a.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(a.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(a.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(a.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(a.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(a.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout."),Object(a.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(a.b)("p",null,"Uses a fake geoip database containing ",Object(a.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"The ",Object(a.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(a.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(a.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(a.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(a.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(a.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(a.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(a.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(a.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(a.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(a.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(a.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(a.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(a.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(a.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(a.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(a.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(a.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(a.b)("h3",{id:"final-stage"},"Final Stage"),Object(a.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(a.b)("h2",{id:"field-names"},"Field names"),Object(a.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(a.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var i=n(0),o=n.n(i);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,o=function(e,t){if(null==e)return{};var n,i,o={},a=Object.keys(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,a=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),u=d(n),p=i,h=u["".concat(s,".").concat(p)]||u[p]||f[p]||a;return n?o.a.createElement(h,r(r({ref:t},l),{},{components:n})):o.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=n.length,s=new Array(a);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<a;l++)s[l]=n[l];return o.a.createElement.apply(null,s)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/6c279c12.cc8d8222.js b/assets/js/6c279c12.225c0b15.js
similarity index 96%
rename from assets/js/6c279c12.cc8d8222.js
rename to assets/js/6c279c12.225c0b15.js
index 8da4305c..01d58913 100644
--- a/assets/js/6c279c12.cc8d8222.js
+++ b/assets/js/6c279c12.225c0b15.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[65],{135:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.5","label":"1.2.5","isLast":false,"docsSidebars":{"version-1.2.5/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.5/pipeline"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.5/pipeline_importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.5/pipeline_logstash"}]},{"collapsed":true,"type":"category","label":"Developer Guides","items":[{"type":"link","label":"Docker Guide","href":"/netsage-pipeline/docs/1.2.5/devel/docker"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.5/deploy/choose_install"},{"type":"link","label":"Installation Guide","href":"/netsage-pipeline/docs/1.2.5/deploy/install"},{"type":"link","label":"Docker Install","href":"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.5/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.5/deploy/choose_install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/deploy/docker_install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/deploy/install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/devel/docker":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/docusaurus":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline_importer":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline_logstash":"version-1.2.5/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[64],{134:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.5","label":"1.2.5","isLast":false,"docsSidebars":{"version-1.2.5/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.5/pipeline"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.5/pipeline_importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.5/pipeline_logstash"}]},{"collapsed":true,"type":"category","label":"Developer Guides","items":[{"type":"link","label":"Docker Guide","href":"/netsage-pipeline/docs/1.2.5/devel/docker"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.5/deploy/choose_install"},{"type":"link","label":"Installation Guide","href":"/netsage-pipeline/docs/1.2.5/deploy/install"},{"type":"link","label":"Docker Install","href":"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.5/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.5/deploy/choose_install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/deploy/docker_install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/deploy/install":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/devel/docker":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/docusaurus":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline_importer":"version-1.2.5/Pipeline","/netsage-pipeline/docs/1.2.5/pipeline_logstash":"version-1.2.5/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/6d913b5d.84d394c4.js b/assets/js/6d913b5d.bb807553.js
similarity index 98%
rename from assets/js/6d913b5d.84d394c4.js
rename to assets/js/6d913b5d.bb807553.js
index 96057e10..28868726 100644
--- a/assets/js/6d913b5d.84d394c4.js
+++ b/assets/js/6d913b5d.bb807553.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[66],{136:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"version-1.2.11/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/versioned_docs/version-1.2.11/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/tag.md",version:"1.2.11",sidebar_label:"Making Releases",sidebar:"version-1.2.11/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[65],{135:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"version-1.2.11/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/versioned_docs/version-1.2.11/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/tag.md",version:"1.2.11",sidebar_label:"Making Releases",sidebar:"version-1.2.11/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/89d5514b.c223f8ef.js b/assets/js/6e2a1c72.7c78a48b.js
similarity index 91%
rename from assets/js/89d5514b.c223f8ef.js
rename to assets/js/6e2a1c72.7c78a48b.js
index f3dd570e..c0499be2 100644
--- a/assets/js/89d5514b.c223f8ef.js
+++ b/assets/js/6e2a1c72.7c78a48b.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[78],{148:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},i={unversionedId:"components/docker_pipeline",id:"version-2.0.0/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-2.0.0/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_pipeline.md",version:"2.0.0"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}l.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[66],{136:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},i={unversionedId:"components/docker_pipeline",id:"version-2.1.0/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-2.1.0/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_pipeline.md",version:"2.1.0"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}l.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/6e3166bc.fa40017d.js b/assets/js/6e3166bc.fa40017d.js
new file mode 100644
index 00000000..b8100605
--- /dev/null
+++ b/assets/js/6e3166bc.fa40017d.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[67],{137:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"version-2.1.0/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/versioned_docs/version-2.1.0/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/tag.md",version:"2.1.0",sidebar_label:"Making Releases",sidebar:"version-2.1.0/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/71e5fd50.9e08f691.js b/assets/js/71e5fd50.9e08f691.js
new file mode 100644
index 00000000..5d32ba6b
--- /dev/null
+++ b/assets/js/71e5fd50.9e08f691.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[68],{138:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-2.1.0/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-2.1.0/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/elastic_search.md",version:"2.1.0",sidebar_label:"Elasticsearch",sidebar:"version-2.1.0/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/pipeline/logstash"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/734dc415.80010e23.js b/assets/js/734dc415.80010e23.js
deleted file mode 100644
index 2180b60c..00000000
--- a/assets/js/734dc415.80010e23.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[67],{137:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/docs/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_simple.md",version:"current",sidebar_label:"Docker Installation",sidebar:"Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/734dc415.9fadde42.js b/assets/js/734dc415.9fadde42.js
new file mode 100644
index 00000000..c5de4a40
--- /dev/null
+++ b/assets/js/734dc415.9fadde42.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[69],{139:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/docs/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_simple.md",version:"current",sidebar_label:"Docker Installation",sidebar:"Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/next/pipeline/elastic"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/799b1578.a774f21c.js b/assets/js/799b1578.ad714f78.js
similarity index 96%
rename from assets/js/799b1578.a774f21c.js
rename to assets/js/799b1578.ad714f78.js
index 4c2be0be..6b4e77a4 100644
--- a/assets/js/799b1578.a774f21c.js
+++ b/assets/js/799b1578.ad714f78.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[68],{138:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.10/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.10/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/importer.md",version:"1.2.10",sidebar_label:"Importer",sidebar:"version-1.2.10/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[70],{140:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.10/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.10/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/importer.md",version:"1.2.10",sidebar_label:"Importer",sidebar:"version-1.2.10/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/7b85b03c.95489b8b.js b/assets/js/7b85b03c.6e3a64bd.js
similarity index 96%
rename from assets/js/7b85b03c.95489b8b.js
rename to assets/js/7b85b03c.6e3a64bd.js
index 077dcca4..3b42f728 100644
--- a/assets/js/7b85b03c.95489b8b.js
+++ b/assets/js/7b85b03c.6e3a64bd.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[69],{139:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.9/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.9/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/nfdump.md",version:"1.2.9",sidebar_label:"Nfdump",sidebar:"version-1.2.9/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[71],{141:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.9/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.9/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/nfdump.md",version:"1.2.9",sidebar_label:"Nfdump",sidebar:"version-1.2.9/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/7c78db09.bc8a300d.js b/assets/js/7c78db09.5ea6e962.js
similarity index 97%
rename from assets/js/7c78db09.bc8a300d.js
rename to assets/js/7c78db09.5ea6e962.js
index 25c93fa0..36764855 100644
--- a/assets/js/7c78db09.bc8a300d.js
+++ b/assets/js/7c78db09.5ea6e962.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[70],{140:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(232)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.11/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",version:"1.2.11",sidebar_label:"Troubleshooting",sidebar:"version-1.2.11/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(i.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Errors:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(i.b)("inlineCode",{parentName:"li"},"docker ps")),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(i.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Disk space:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(i.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(i.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(i.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Memory:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return d})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(o),b=n,f=d["".concat(a,".").concat(b)]||d[b]||p[b]||i;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[72],{142:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(228)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.11/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",version:"1.2.11",sidebar_label:"Troubleshooting",sidebar:"version-1.2.11/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(i.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Errors:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(i.b)("inlineCode",{parentName:"li"},"docker ps")),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(i.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Disk space:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(i.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(i.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(i.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Memory:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return d})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(o),b=n,f=d["".concat(a,".").concat(b)]||d[b]||p[b]||i;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/7d0b7755.4b75443f.js b/assets/js/7d0b7755.4b75443f.js
deleted file mode 100644
index 0dca27e7..00000000
--- a/assets/js/7d0b7755.4b75443f.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[71],{141:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/docs/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/next/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/elastic_search.md",version:"current",sidebar_label:"Elasticsearch",sidebar:"Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/next/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/next/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/7d0b7755.c19774b1.js b/assets/js/7d0b7755.c19774b1.js
new file mode 100644
index 00000000..cddff5bc
--- /dev/null
+++ b/assets/js/7d0b7755.c19774b1.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[73],{143:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/docs/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/next/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/elastic_search.md",version:"current",sidebar_label:"Elasticsearch",sidebar:"Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/next/pipeline/logstash"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/7d380c4c.ad3c0101.js b/assets/js/7d380c4c.b92baa88.js
similarity index 96%
rename from assets/js/7d380c4c.ad3c0101.js
rename to assets/js/7d380c4c.b92baa88.js
index 544abdc3..02151b05 100644
--- a/assets/js/7d380c4c.ad3c0101.js
+++ b/assets/js/7d380c4c.b92baa88.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[72],{142:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.6","label":"1.2.6","isLast":false,"docsSidebars":{"version-1.2.6/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.6/pipeline"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.6/pipeline_importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.6/pipeline_logstash"}]},{"collapsed":true,"type":"category","label":"Developer Guides","items":[{"type":"link","label":"Docker Guide","href":"/netsage-pipeline/docs/1.2.6/devel/docker"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.6/deploy/choose_install"},{"type":"link","label":"Installation Guide","href":"/netsage-pipeline/docs/1.2.6/deploy/install"},{"type":"link","label":"Docker Install","href":"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.6/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.6/deploy/choose_install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/deploy/docker_install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/deploy/install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/devel/docker":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/docusaurus":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline_importer":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline_logstash":"version-1.2.6/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[74],{144:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.6","label":"1.2.6","isLast":false,"docsSidebars":{"version-1.2.6/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.6/pipeline"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.6/pipeline_importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.6/pipeline_logstash"}]},{"collapsed":true,"type":"category","label":"Developer Guides","items":[{"type":"link","label":"Docker Guide","href":"/netsage-pipeline/docs/1.2.6/devel/docker"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.6/deploy/choose_install"},{"type":"link","label":"Installation Guide","href":"/netsage-pipeline/docs/1.2.6/deploy/install"},{"type":"link","label":"Docker Install","href":"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.6/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.6/deploy/choose_install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/deploy/docker_install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/deploy/install":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/devel/docker":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/docusaurus":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline_importer":"version-1.2.6/Pipeline","/netsage-pipeline/docs/1.2.6/pipeline_logstash":"version-1.2.6/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/8084fbd4.5592d29d.js b/assets/js/8084fbd4.5592d29d.js
deleted file mode 100644
index 70ed3b66..00000000
--- a/assets/js/8084fbd4.5592d29d.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[73],{143:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-2.0.0/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-2.0.0/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/tstat.md",version:"2.0.0",sidebar_label:"Tstat Data",sidebar:"version-2.0.0/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/832ca04e.54bca9dc.js b/assets/js/832ca04e.24044575.js
similarity index 95%
rename from assets/js/832ca04e.54bca9dc.js
rename to assets/js/832ca04e.24044575.js
index 566e67d7..907c0190 100644
--- a/assets/js/832ca04e.54bca9dc.js
+++ b/assets/js/832ca04e.24044575.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[74],{144:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.8/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.8/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/tstat.md",version:"1.2.8",sidebar_label:"Tstat",sidebar:"version-1.2.8/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[75],{145:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.8/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.8/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/tstat.md",version:"1.2.8",sidebar_label:"Tstat",sidebar:"version-1.2.8/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/83dbc9a5.e629373c.js b/assets/js/83dbc9a5.4a4e5652.js
similarity index 98%
rename from assets/js/83dbc9a5.e629373c.js
rename to assets/js/83dbc9a5.4a4e5652.js
index 5908f351..203163ba 100644
--- a/assets/js/83dbc9a5.e629373c.js
+++ b/assets/js/83dbc9a5.4a4e5652.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[75],{145:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),i=(n(0),n(232)),a={id:"docker",title:"Docker Guide",sidebar_label:"Docker Guide"},l={unversionedId:"devel/docker",id:"version-1.2.5/devel/docker",isDocsHomePage:!1,title:"Docker Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.5/devel/docker.md",slug:"/devel/docker",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/devel/docker.md",version:"1.2.5",sidebar_label:"Docker Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],c={toc:s};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(i.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(i.b)("p",null,"To select a released version please do the following."),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh tagValue")),Object(i.b)("p",null,"Example:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh v1.2.5")),Object(i.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(i.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(i.b)("h3",{id:"development-version"},"Development Version"),Object(i.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(i.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(i.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n")),Object(i.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(i.b)("h3",{id:"rabbit"},"Rabbit"),Object(i.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(i.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(i.b)("h3",{id:"importer"},"Importer"),Object(i.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(i.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(i.b)("h3",{id:"logstash"},"Logstash"),Object(i.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(i.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(i.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line:  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d")))),Object(i.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),b=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=b(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=b(n),d=o,h=p["".concat(a,".").concat(d)]||p[d]||u[d]||i;return n?r.a.createElement(h,l(l({ref:t},c),{},{components:n})):r.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[76],{146:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),i=(n(0),n(228)),a={id:"docker",title:"Docker Guide",sidebar_label:"Docker Guide"},l={unversionedId:"devel/docker",id:"version-1.2.5/devel/docker",isDocsHomePage:!1,title:"Docker Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.5/devel/docker.md",slug:"/devel/docker",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/devel/docker.md",version:"1.2.5",sidebar_label:"Docker Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],c={toc:s};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(i.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(i.b)("p",null,"To select a released version please do the following."),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh tagValue")),Object(i.b)("p",null,"Example:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh v1.2.5")),Object(i.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(i.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(i.b)("h3",{id:"development-version"},"Development Version"),Object(i.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(i.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(i.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n")),Object(i.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(i.b)("h3",{id:"rabbit"},"Rabbit"),Object(i.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(i.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(i.b)("h3",{id:"importer"},"Importer"),Object(i.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(i.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(i.b)("h3",{id:"logstash"},"Logstash"),Object(i.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(i.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(i.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line:  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d")))),Object(i.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),b=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=b(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=b(n),d=o,h=p["".concat(a,".").concat(d)]||p[d]||u[d]||i;return n?r.a.createElement(h,l(l({ref:t},c),{},{components:n})):r.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/86d09e6e.43e743b3.js b/assets/js/86d09e6e.43e743b3.js
new file mode 100644
index 00000000..aabdb369
--- /dev/null
+++ b/assets/js/86d09e6e.43e743b3.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[77],{147:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(228)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-2.1.0/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-2.1.0/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_troubleshooting.md",version:"2.1.0",sidebar_label:"Troubleshooting",sidebar:"version-2.1.0/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file (you may need to install nfdump).'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs $service"),", where $service is logstash, importer, rabbit, etc."),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.")),Object(i.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Errors:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"See if any of the containers has died using  ",Object(i.b)("inlineCode",{parentName:"li"},"docker ps")),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0Eg, ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs logstash"),"."),Object(i.b)("li",{parentName:"ul"},"If logstash dies with an error about not finding ","*",".conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Disk space:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(i.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(i.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(i.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). ")),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Memory:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.")))}u.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return d})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},d=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(o),b=n,f=d["".concat(a,".").concat(b)]||d[b]||p[b]||i;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/879923b2.2bf3c135.js b/assets/js/879923b2.90f3d9b4.js
similarity index 99%
rename from assets/js/879923b2.2bf3c135.js
rename to assets/js/879923b2.90f3d9b4.js
index a7bd33f2..c4e49d1e 100644
--- a/assets/js/879923b2.2bf3c135.js
+++ b/assets/js/879923b2.90f3d9b4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[76],{146:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(232)),r={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.7/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:'In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see https://github.com/netsage-project/docker-nfdump-collector) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read "Docker Advanced" after reading through this guide.',source:"@site/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",version:"1.2.7",sidebar_label:"Docker Simple",sidebar:"version-1.2.7/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"}},c=[{value:"First",id:"first",children:[]},{value:"Docker-compose.override.yml",id:"docker-composeoverrideyml",children:[]},{value:"Environment File",id:"environment-file",children:[]},{value:"Pipeline Version",id:"pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[]},{value:"Running the Pipeline",id:"running-the-pipeline",children:[]},{value:"Data sources",id:"data-sources",children:[]},{value:"Upgrading",id:"upgrading",children:[]}],s={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"https://github.com/netsage-project/docker-nfdump-collector"),') and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read "Docker Advanced" after reading through this guide.'),Object(i.b)("h3",{id:"first"},"First"),Object(i.b)("p",null,"If you haven't already, install Docker/compose and clone this project from github (",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline.git"},"https://github.com/netsage-project/netsage-pipeline.git"),")."),Object(i.b)("h3",{id:"docker-composeoverrideyml"},"Docker-compose.override.yml"),Object(i.b)("p",null,"The pattern for running the Pipeline, with docker-based collectors, is defined in the docker-compose.override_example.yml. Copy this to docker-compose.override.yml. "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(i.b)("p",null,"By default this will bring up a single netflow collector and a single sflow collector. For most people this is more than enough. If you're sticking to the default you don't need to make any changes to the docker-compose.override_example.yml"),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"If you are only interested in netflow OR sflow data, you should remove the section for the collector that is not used."))),Object(i.b)("p",null,"This file also specifies port numbers and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/")," . Each collector is namespaced by its type so the sflow collector will write data to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,Object(i.b)("p",{parentName:"p"},"Please copy ",Object(i.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(i.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(i.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},'sflowSensorName="my sflow sensor name"\nnetflowSensorName="my netflow sensor name"\n')),Object(i.b)("p",{parentName:"p"},"Simply change the names to unique identifiers and you're good to go. (Use quotes if the names have spaces.) "),Object(i.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(i.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and can be used in visualizations. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet Sflow" and "RNDNet Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(i.b)("ul",{parentName:"p"},Object(i.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used."),Object(i.b)("li",{parentName:"ul"},"If you have only one collector, comment out the line for the one you are not using."),Object(i.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.'),Object(i.b)("li",{parentName:"ul"},"If you're not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don't start up either collector.")),Object(i.b)("p",{parentName:"p"},"Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware."),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  Use the default ",Object(i.b)("inlineCode",{parentName:"p"},"rabbit")," for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(i.b)("p",{parentName:"p"},"The Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(i.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(i.b)("h3",{id:"pipeline-version"},"Pipeline Version"),Object(i.b)("p",null,"Once you've created the docker-compose.override.xml and finished adjusting it for any customizations, you're ready to select your version."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},'git fetch\ngit checkout "tag name"\n./scripts/docker_select_version.sh\n')),Object(i.b)("p",null,'Replace "tag name" with the version you intend to use, e.g., "v1.2.5". Select the same version when prompted by docker_select_version.sh.'),Object(i.b)("h3",{id:"running-the-collectors"},"Running the Collectors"),Object(i.b)("p",null,"After selecting the version to run, you can start the two flow collectors by running the following line. If you only need one of the collectors, remove the other from this command."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(i.b)("h3",{id:"running-the-pipeline"},"Running the Pipeline"),Object(i.b)("p",null,Object(i.b)("p",{parentName:"p"},"Start up the pipeline (importers and logstash) using:"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(i.b)("p",{parentName:"p"},"You can check the logs for each of the container by running"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(i.b)("p",{parentName:"p"},"Shut down the pipeline using:"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"sflow "),Object(i.b)("li",{parentName:"ul"},"netflow"),Object(i.b)("li",{parentName:"ul"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data. "),Object(i.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where nfcapd and/or sfcapd collectors are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data. (See the Docker Advanced guide.)"),Object(i.b)("h2",{id:"upgrading"},"Upgrading"),Object(i.b)("p",null,Object(i.b)("h3",{parentName:"p"},"Update Source Code"),Object(i.b)("p",{parentName:"p"},"To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  "),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(i.b)("h3",{parentName:"p"},"Docker and Collectors"),Object(i.b)("p",{parentName:"p"},"Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, ",Object(i.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),"). Also check the docker version (eg, ",Object(i.b)("inlineCode",{parentName:"p"},'version: "3.7"'),") to see if you'll need to ugrade docker."),Object(i.b)("h3",{parentName:"p"},"Select Release Version"),Object(i.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},'git checkout "tag_value" \n./scripts/docker_select_version.sh\n')),Object(i.b)("h3",{parentName:"p"},"Update docker containers"),Object(i.b)("p",{parentName:"p"},"This applies for both development and release"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre"},"docker-compose pull\n"))))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),p=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=p(n),b=o,h=d["".concat(r,".").concat(b)]||d[b]||m[b]||i;return n?a.a.createElement(h,l(l({ref:t},s),{},{components:n})):a.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var s=2;s<i;s++)r[s]=n[s];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[78],{148:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(228)),r={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.7/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:'In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see https://github.com/netsage-project/docker-nfdump-collector) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read "Docker Advanced" after reading through this guide.',source:"@site/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",version:"1.2.7",sidebar_label:"Docker Simple",sidebar:"version-1.2.7/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"}},c=[{value:"First",id:"first",children:[]},{value:"Docker-compose.override.yml",id:"docker-composeoverrideyml",children:[]},{value:"Environment File",id:"environment-file",children:[]},{value:"Pipeline Version",id:"pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[]},{value:"Running the Pipeline",id:"running-the-pipeline",children:[]},{value:"Data sources",id:"data-sources",children:[]},{value:"Upgrading",id:"upgrading",children:[]}],s={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"https://github.com/netsage-project/docker-nfdump-collector"),') and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read "Docker Advanced" after reading through this guide.'),Object(i.b)("h3",{id:"first"},"First"),Object(i.b)("p",null,"If you haven't already, install Docker/compose and clone this project from github (",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline.git"},"https://github.com/netsage-project/netsage-pipeline.git"),")."),Object(i.b)("h3",{id:"docker-composeoverrideyml"},"Docker-compose.override.yml"),Object(i.b)("p",null,"The pattern for running the Pipeline, with docker-based collectors, is defined in the docker-compose.override_example.yml. Copy this to docker-compose.override.yml. "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(i.b)("p",null,"By default this will bring up a single netflow collector and a single sflow collector. For most people this is more than enough. If you're sticking to the default you don't need to make any changes to the docker-compose.override_example.yml"),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"If you are only interested in netflow OR sflow data, you should remove the section for the collector that is not used."))),Object(i.b)("p",null,"This file also specifies port numbers and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/")," . Each collector is namespaced by its type so the sflow collector will write data to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(i.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,Object(i.b)("p",{parentName:"p"},"Please copy ",Object(i.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(i.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(i.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},'sflowSensorName="my sflow sensor name"\nnetflowSensorName="my netflow sensor name"\n')),Object(i.b)("p",{parentName:"p"},"Simply change the names to unique identifiers and you're good to go. (Use quotes if the names have spaces.) "),Object(i.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(i.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and can be used in visualizations. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet Sflow" and "RNDNet Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(i.b)("ul",{parentName:"p"},Object(i.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used."),Object(i.b)("li",{parentName:"ul"},"If you have only one collector, comment out the line for the one you are not using."),Object(i.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.'),Object(i.b)("li",{parentName:"ul"},"If you're not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don't start up either collector.")),Object(i.b)("p",{parentName:"p"},"Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware."),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  Use the default ",Object(i.b)("inlineCode",{parentName:"p"},"rabbit")," for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(i.b)("p",{parentName:"p"},"The Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(i.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(i.b)("p",{parentName:"p"},Object(i.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(i.b)("h3",{id:"pipeline-version"},"Pipeline Version"),Object(i.b)("p",null,"Once you've created the docker-compose.override.xml and finished adjusting it for any customizations, you're ready to select your version."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},'git fetch\ngit checkout "tag name"\n./scripts/docker_select_version.sh\n')),Object(i.b)("p",null,'Replace "tag name" with the version you intend to use, e.g., "v1.2.5". Select the same version when prompted by docker_select_version.sh.'),Object(i.b)("h3",{id:"running-the-collectors"},"Running the Collectors"),Object(i.b)("p",null,"After selecting the version to run, you can start the two flow collectors by running the following line. If you only need one of the collectors, remove the other from this command."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(i.b)("h3",{id:"running-the-pipeline"},"Running the Pipeline"),Object(i.b)("p",null,Object(i.b)("p",{parentName:"p"},"Start up the pipeline (importers and logstash) using:"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(i.b)("p",{parentName:"p"},"You can check the logs for each of the container by running"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(i.b)("p",{parentName:"p"},"Shut down the pipeline using:"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"sflow "),Object(i.b)("li",{parentName:"ul"},"netflow"),Object(i.b)("li",{parentName:"ul"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data. "),Object(i.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where nfcapd and/or sfcapd collectors are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data. (See the Docker Advanced guide.)"),Object(i.b)("h2",{id:"upgrading"},"Upgrading"),Object(i.b)("p",null,Object(i.b)("h3",{parentName:"p"},"Update Source Code"),Object(i.b)("p",{parentName:"p"},"To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  "),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(i.b)("h3",{parentName:"p"},"Docker and Collectors"),Object(i.b)("p",{parentName:"p"},"Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, ",Object(i.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:1.6.18"),"). Also check the docker version (eg, ",Object(i.b)("inlineCode",{parentName:"p"},'version: "3.7"'),") to see if you'll need to ugrade docker."),Object(i.b)("h3",{parentName:"p"},"Select Release Version"),Object(i.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre",className:"language-sh"},'git checkout "tag_value" \n./scripts/docker_select_version.sh\n')),Object(i.b)("h3",{parentName:"p"},"Update docker containers"),Object(i.b)("p",{parentName:"p"},"This applies for both development and release"),Object(i.b)("pre",{parentName:"p"},Object(i.b)("code",{parentName:"pre"},"docker-compose pull\n"))))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),p=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=p(n),b=o,h=d["".concat(r,".").concat(b)]||d[b]||m[b]||i;return n?a.a.createElement(h,l(l({ref:t},s),{},{components:n})):a.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var s=2;s<i;s++)r[s]=n[s];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/8997819f.64d4557b.js b/assets/js/8997819f.aa7254b4.js
similarity index 97%
rename from assets/js/8997819f.64d4557b.js
rename to assets/js/8997819f.aa7254b4.js
index 4bb50f03..acc6cc76 100644
--- a/assets/js/8997819f.64d4557b.js
+++ b/assets/js/8997819f.aa7254b4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[77],{147:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.11/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/versioned_docs/version-1.2.11/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_env.md",version:"1.2.11"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,s(s({ref:t},l),{},{components:n})):a.a.createElement(d,s({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[79],{149:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.11/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/versioned_docs/version-1.2.11/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_env.md",version:"1.2.11"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,s(s({ref:t},l),{},{components:n})):a.a.createElement(d,s({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9b6d3056.ff7822d9.js b/assets/js/8b301107.d02fcb00.js
similarity index 95%
rename from assets/js/9b6d3056.ff7822d9.js
rename to assets/js/8b301107.d02fcb00.js
index 592866c0..9fa595a6 100644
--- a/assets/js/9b6d3056.ff7822d9.js
+++ b/assets/js/8b301107.d02fcb00.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[91],{161:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-2.0.0/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/versioned_docs/version-2.0.0/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_env.md",version:"2.0.0"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,s(s({ref:t},l),{},{components:n})):a.a.createElement(d,s({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[80],{150:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-2.1.0/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Next, copy env.example to .env",source:"@site/versioned_docs/version-2.1.0/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_env.md",version:"2.1.0"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",null,"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",null,"The following options are described in the Docker Advanced section:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return d}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),p=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},b=function(e){var t=p(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},m=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),b=p(n),m=o,d=b["".concat(i,".").concat(m)]||b[m]||u[m]||r;return n?a.a.createElement(d,s(s({ref:t},l),{},{components:n})):a.a.createElement(d,s({ref:t},l))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/8bbfcfa9.f0501de3.js b/assets/js/8bbfcfa9.dc015f8e.js
similarity index 96%
rename from assets/js/8bbfcfa9.f0501de3.js
rename to assets/js/8bbfcfa9.dc015f8e.js
index 126cd095..bde98bc8 100644
--- a/assets/js/8bbfcfa9.f0501de3.js
+++ b/assets/js/8bbfcfa9.dc015f8e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[79],{149:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.10/deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/versioned_docs/version-1.2.10/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/choosing.md",version:"1.2.10",sidebar_label:"Choose Install",sidebar:"version-1.2.10/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[81],{151:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.10/deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/versioned_docs/version-1.2.10/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/choosing.md",version:"1.2.10",sidebar_label:"Choose Install",sidebar:"version-1.2.10/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9001d127.1e177d17.js b/assets/js/9001d127.1e177d17.js
deleted file mode 100644
index 348bd660..00000000
--- a/assets/js/9001d127.1e177d17.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[80],{150:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return p}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-2.0.0/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-2.0.0/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/elastic_search.md",version:"2.0.0",sidebar_label:"Elasticsearch",sidebar:"version-2.0.0/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/pipeline/logstash"},next:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function p(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f0542..."),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@pipeline_ver"),Object(l.b)("td",{parentName:"tr",align:null},"1.2.11"),Object(l.b)("td",{parentName:"tr",align:null},"Version number of the pipeline used to process this flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}p.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return s})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),p=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},s=function(t){var e=p(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),s=p(a),o=n,u=s["".concat(b,".").concat(o)]||s[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/906738b8.33482c0f.js b/assets/js/906738b8.3a283baa.js
similarity index 97%
rename from assets/js/906738b8.33482c0f.js
rename to assets/js/906738b8.3a283baa.js
index 7b65f090..3ec7a7a6 100644
--- a/assets/js/906738b8.33482c0f.js
+++ b/assets/js/906738b8.3a283baa.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[81],{151:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.7","label":"1.2.7","isLast":false,"docsSidebars":{"version-1.2.7/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.7/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.7/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.7/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.7/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.7/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.7/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/choose_install":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/devel/dev_dataset":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/docusaurus":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/elastic":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/importer":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/intro":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/logstash":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/nfdump":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/tstat":"version-1.2.7/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[82],{152:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.7","label":"1.2.7","isLast":false,"docsSidebars":{"version-1.2.7/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.7/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.7/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.7/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.7/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.7/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.7/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.7/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/choose_install":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/devel/dev_dataset":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/docusaurus":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/elastic":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/importer":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/intro":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/logstash":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/nfdump":"version-1.2.7/Pipeline","/netsage-pipeline/docs/1.2.7/pipeline/tstat":"version-1.2.7/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/9191c864.9b61c372.js b/assets/js/9191c864.9b61c372.js
new file mode 100644
index 00000000..dc40b26e
--- /dev/null
+++ b/assets/js/9191c864.9b61c372.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[83],{153:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},l={unversionedId:"deploy/docker_upgrade",id:"version-2.1.0/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-2.1.0/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_upgrade.md",version:"2.1.0",sidebar_label:"Docker - Upgrading",sidebar:"version-2.1.0/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot"}},c=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:c};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit fetch\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,"Run these commands to upgrade to the current announced version. While older versions of the pipeline are available, they are not recommended or supported for daily use. If in doubt, the Master branch contains the latest version. The documentation version at the top of this page is also the current pipeline version number."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git switch {tag} \ngit pull\n./Upgrade_x_x_x.sh (where x_x_x is the current version you are upgrading to)\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,l(l({ref:t},s),{},{components:n})):o.a.createElement(m,l({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/935f2afb.5c9a98e4.js b/assets/js/935f2afb.5c9a98e4.js
deleted file mode 100644
index ef703a8f..00000000
--- a/assets/js/935f2afb.5c9a98e4.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[82],{152:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"master (unreleased)","isLast":false,"docsSidebars":{"Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/next/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/next/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/next/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/next/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/next/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/next/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/next/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/next/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/next/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/next/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/next/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/next/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/next/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/next/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/next/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/next/deploy/bare_metal_install":"Pipeline","/netsage-pipeline/docs/next/deploy/choose_install":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_install_advanced":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_install_simple":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_troubleshoot":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_upgrade":"Pipeline","/netsage-pipeline/docs/next/devel/dev_dataset":"Pipeline","/netsage-pipeline/docs/next/devel/docker_dev_guide":"Pipeline","/netsage-pipeline/docs/next/devel/docker_dev_tag":"Pipeline","/netsage-pipeline/docs/next/devel/docusaurus":"Pipeline","/netsage-pipeline/docs/next/pipeline/elastic":"Pipeline","/netsage-pipeline/docs/next/pipeline/importer":"Pipeline","/netsage-pipeline/docs/next/pipeline/intro":"Pipeline","/netsage-pipeline/docs/next/pipeline/logstash":"Pipeline","/netsage-pipeline/docs/next/pipeline/nfdump":"Pipeline","/netsage-pipeline/docs/next/pipeline/tstat":"Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/935f2afb.9f248d32.js b/assets/js/935f2afb.9f248d32.js
new file mode 100644
index 00000000..e3f14c4c
--- /dev/null
+++ b/assets/js/935f2afb.9f248d32.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[84],{154:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"master (unreleased)","isLast":false,"docsSidebars":{"Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/next/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/next/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/next/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/next/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/next/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/next/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/next/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/next/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/next/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/next/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/next/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/next/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/next/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/next/deploy/docker_install_advanced":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_install_simple":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_troubleshoot":"Pipeline","/netsage-pipeline/docs/next/deploy/docker_upgrade":"Pipeline","/netsage-pipeline/docs/next/devel/dev_dataset":"Pipeline","/netsage-pipeline/docs/next/devel/docker_dev_guide":"Pipeline","/netsage-pipeline/docs/next/devel/docker_dev_tag":"Pipeline","/netsage-pipeline/docs/next/devel/docusaurus":"Pipeline","/netsage-pipeline/docs/next/pipeline/elastic":"Pipeline","/netsage-pipeline/docs/next/pipeline/importer":"Pipeline","/netsage-pipeline/docs/next/pipeline/intro":"Pipeline","/netsage-pipeline/docs/next/pipeline/logstash":"Pipeline","/netsage-pipeline/docs/next/pipeline/nfdump":"Pipeline","/netsage-pipeline/docs/next/pipeline/tstat":"Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/96852331.87e388ce.js b/assets/js/96852331.822f904f.js
similarity index 96%
rename from assets/js/96852331.87e388ce.js
rename to assets/js/96852331.822f904f.js
index c811a186..c627e8ba 100644
--- a/assets/js/96852331.87e388ce.js
+++ b/assets/js/96852331.822f904f.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[83],{153:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.11/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.11/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/importer.md",version:"1.2.11",sidebar_label:"Importer",sidebar:"version-1.2.11/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[85],{155:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.11/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.11/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/importer.md",version:"1.2.11",sidebar_label:"Importer",sidebar:"version-1.2.11/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9720d299.d4ac0286.js b/assets/js/9720d299.9abe8149.js
similarity index 98%
rename from assets/js/9720d299.d4ac0286.js
rename to assets/js/9720d299.9abe8149.js
index 646df775..b5810d37 100644
--- a/assets/js/9720d299.d4ac0286.js
+++ b/assets/js/9720d299.9abe8149.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[84],{154:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),o=n(7),a=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.9/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.9/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/logstash.md",version:"1.2.9",sidebar_label:"Logstash",sidebar:"version-1.2.9/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(a.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(a.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(a.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(a.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(a.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(a.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(a.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(a.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(a.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(a.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(a.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(a.b)("p",null,"Makes any changes to fields needed for specific sensors. It currently provides the ability to change the sensor name for flows from a specified sensor which go through a certain interface, and the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags are set to False by default."),Object(a.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(a.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(a.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(a.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(a.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(a.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(a.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(a.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(a.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(a.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(a.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(a.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(a.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(a.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(a.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(a.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(a.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(a.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(a.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(a.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(a.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(a.b)("p",null,"Uses a fake geoip database containing ",Object(a.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"The ",Object(a.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(a.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(a.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(a.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(a.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(a.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(a.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(a.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(a.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(a.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(a.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(a.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(a.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(a.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(a.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(a.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(a.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(a.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(a.b)("h3",{id:"final-stage"},"Final Stage"),Object(a.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(a.b)("h2",{id:"field-names"},"Field names"),Object(a.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(a.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),o=n.n(i);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,o=function(e,t){if(null==e)return{};var n,i,o={},a=Object.keys(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),d=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||a;return n?o.a.createElement(h,r(r({ref:t},c),{},{components:n})):o.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=n.length,s=new Array(a);s[0]=p;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return o.a.createElement.apply(null,s)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[86],{156:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),o=n(7),a=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.9/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.9/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/logstash.md",version:"1.2.9",sidebar_label:"Logstash",sidebar:"version-1.2.9/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(a.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(a.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(a.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(a.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(a.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(a.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(a.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(a.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(a.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(a.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(a.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(a.b)("p",null,"Makes any changes to fields needed for specific sensors. It currently provides the ability to change the sensor name for flows from a specified sensor which go through a certain interface, and the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags are set to False by default."),Object(a.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(a.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(a.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(a.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(a.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(a.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(a.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(a.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(a.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(a.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(a.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(a.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(a.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(a.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(a.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(a.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(a.b)("p",null,Object(a.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(a.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(a.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(a.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(a.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(a.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(a.b)("p",null,"Uses a fake geoip database containing ",Object(a.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(a.b)("p",null,"Notes: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"The ",Object(a.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(a.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(a.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(a.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(a.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(a.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(a.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(a.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(a.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(a.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(a.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(a.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(a.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(a.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(a.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(a.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(a.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(a.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(a.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(a.b)("h3",{id:"final-stage"},"Final Stage"),Object(a.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(a.b)("h2",{id:"field-names"},"Field names"),Object(a.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(a.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),o=n.n(i);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,o=function(e,t){if(null==e)return{};var n,i,o={},a=Object.keys(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i<a.length;i++)n=a[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),d=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||a;return n?o.a.createElement(h,r(r({ref:t},c),{},{components:n})):o.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=n.length,s=new Array(a);s[0]=p;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return o.a.createElement.apply(null,s)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/97edd56e.3b71c751.js b/assets/js/97edd56e.d5ab66b5.js
similarity index 98%
rename from assets/js/97edd56e.3b71c751.js
rename to assets/js/97edd56e.d5ab66b5.js
index e3a6a00c..84b51e29 100644
--- a/assets/js/97edd56e.3b71c751.js
+++ b/assets/js/97edd56e.d5ab66b5.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[85],{155:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.7/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.7/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/docker.md",version:"1.2.7",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.7/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[87],{157:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.7/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.7/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/docker.md",version:"1.2.7",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.7/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/992819d4.508016b4.js b/assets/js/992819d4.2993e926.js
similarity index 97%
rename from assets/js/992819d4.508016b4.js
rename to assets/js/992819d4.2993e926.js
index b770228d..b156f287 100644
--- a/assets/js/992819d4.508016b4.js
+++ b/assets/js/992819d4.2993e926.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[86],{156:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.8","label":"1.2.8","isLast":false,"docsSidebars":{"version-1.2.8/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.8/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.8/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.8/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.8/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.8/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.8/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/choose_install":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/devel/dev_dataset":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/docusaurus":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/elastic":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/importer":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/intro":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/logstash":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/nfdump":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/tstat":"version-1.2.8/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[88],{158:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.8","label":"1.2.8","isLast":false,"docsSidebars":{"version-1.2.8/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.8/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.8/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.8/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.8/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.8/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.8/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/choose_install":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/devel/dev_dataset":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/docusaurus":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/elastic":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/importer":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/intro":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/logstash":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/nfdump":"version-1.2.8/Pipeline","/netsage-pipeline/docs/1.2.8/pipeline/tstat":"version-1.2.8/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/99d6017e.6764251a.js b/assets/js/99d6017e.cb54715a.js
similarity index 96%
rename from assets/js/99d6017e.6764251a.js
rename to assets/js/99d6017e.cb54715a.js
index 2f4080ff..463b84a6 100644
--- a/assets/js/99d6017e.6764251a.js
+++ b/assets/js/99d6017e.cb54715a.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[87],{157:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.9/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.9/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/importer.md",version:"1.2.9",sidebar_label:"Importer",sidebar:"version-1.2.9/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[89],{159:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.9/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.9/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/importer.md",version:"1.2.9",sidebar_label:"Importer",sidebar:"version-1.2.9/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9a5c9747.5a3e990f.js b/assets/js/9a5c9747.85ad9bca.js
similarity index 96%
rename from assets/js/9a5c9747.5a3e990f.js
rename to assets/js/9a5c9747.85ad9bca.js
index 28286836..f780d9be 100644
--- a/assets/js/9a5c9747.5a3e990f.js
+++ b/assets/js/9a5c9747.85ad9bca.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[88],{158:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.11/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",version:"1.2.11",sidebar_label:"Replay Dataset",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[90],{160:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.11/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",version:"1.2.11",sidebar_label:"Replay Dataset",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9aa54785.d3836c06.js b/assets/js/9aa54785.d3836c06.js
deleted file mode 100644
index 1e91ded5..00000000
--- a/assets/js/9aa54785.d3836c06.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[89],{159:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"2.0.0","label":"2.0.0","isLast":true,"docsSidebars":{"version-2.0.0/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/deploy/docker_install_advanced"},{"type":"link","label":"Docker - Upgrading","href":"/netsage-pipeline/docs/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/devel/docusaurus"},{"type":"link","label":"Making Releases","href":"/netsage-pipeline/docs/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/deploy/bare_metal_install":"version-2.0.0/Pipeline","/netsage-pipeline/docs/deploy/choose_install":"version-2.0.0/Pipeline","/netsage-pipeline/docs/deploy/docker_install_advanced":"version-2.0.0/Pipeline","/netsage-pipeline/docs/deploy/docker_install_simple":"version-2.0.0/Pipeline","/netsage-pipeline/docs/deploy/docker_troubleshoot":"version-2.0.0/Pipeline","/netsage-pipeline/docs/deploy/docker_upgrade":"version-2.0.0/Pipeline","/netsage-pipeline/docs/devel/dev_dataset":"version-2.0.0/Pipeline","/netsage-pipeline/docs/devel/docker_dev_guide":"version-2.0.0/Pipeline","/netsage-pipeline/docs/devel/docker_dev_tag":"version-2.0.0/Pipeline","/netsage-pipeline/docs/devel/docusaurus":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/elastic":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/importer":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/intro":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/logstash":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/nfdump":"version-2.0.0/Pipeline","/netsage-pipeline/docs/pipeline/tstat":"version-2.0.0/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/9b505a1f.0d4ca405.js b/assets/js/9b505a1f.f4b85bdc.js
similarity index 98%
rename from assets/js/9b505a1f.0d4ca405.js
rename to assets/js/9b505a1f.f4b85bdc.js
index 32375b82..3a60290e 100644
--- a/assets/js/9b505a1f.0d4ca405.js
+++ b/assets/js/9b505a1f.f4b85bdc.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[90],{160:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.9/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.9/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/docker.md",version:"1.2.9",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.9/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[91],{161:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),a=(n(0),n(228)),i={id:"docker_dev_guide",title:"Docker Dev Guide",sidebar_label:"Docker Dev Guide"},l={unversionedId:"devel/docker_dev_guide",id:"version-1.2.9/devel/docker_dev_guide",isDocsHomePage:!1,title:"Docker Dev Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.9/devel/docker.md",slug:"/devel/docker_dev_guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/docker.md",version:"1.2.9",sidebar_label:"Docker Dev Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.9/docusaurus"}},c=[{value:"Configure Collectors",id:"configure-collectors",children:[]},{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],s={toc:c};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(a.b)("h2",{id:"configure-collectors"},"Configure Collectors"),Object(a.b)("p",null,"Before you start if you haven't already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},"Docker Advanced Install Guide")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(a.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(a.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(a.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(a.b)("p",null,"To select a released version please do the following."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"}," scripts/docker_select_version.sh tagValue \n")),Object(a.b)("p",null,"Example:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"scripts/docker_select_version.sh v1.2.5 \n\n")),Object(a.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(a.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(a.b)("h3",{id:"development-version"},"Development Version"),Object(a.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(a.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(a.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(a.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(a.b)("p",null,"If you would like to build the ",Object(a.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n\n")),Object(a.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(a.b)("h3",{id:"environment-file"},"Environment File"),Object(a.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(a.b)("h3",{id:"rabbit"},"Rabbit"),Object(a.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(a.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(a.b)("h3",{id:"importer"},"Importer"),Object(a.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(a.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(a.b)("h3",{id:"logstash"},"Logstash"),Object(a.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(a.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(a.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(a.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(a.b)("ol",{start:2},Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Comment out the ",Object(a.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(a.b)("li",{parentName:"ol"},Object(a.b)("p",{parentName:"li"},"Run the containers using the following line: ",Object(a.b)("inlineCode",{parentName:"p"}," "),"  ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d"),"  ",Object(a.b)("inlineCode",{parentName:"p"}," ")))),Object(a.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(a.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(a.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(a.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(a.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=r.a.createContext({}),b=function(e){var t=r.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=b(e.components);return r.a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),u=b(n),d=o,h=u["".concat(i,".").concat(d)]||u[d]||p[d]||a;return n?r.a.createElement(h,l(l({ref:t},s),{},{components:n})):r.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=n[s];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9bfdba8c.baa4761b.js b/assets/js/9bfdba8c.b2391bbd.js
similarity index 97%
rename from assets/js/9bfdba8c.baa4761b.js
rename to assets/js/9bfdba8c.b2391bbd.js
index 871a16c7..8c1073af 100644
--- a/assets/js/9bfdba8c.baa4761b.js
+++ b/assets/js/9bfdba8c.b2391bbd.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[92],{162:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.8/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.8/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_first_steps.md",version:"1.2.8"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[92],{162:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.8/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.8/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_first_steps.md",version:"1.2.8"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9d5eac8e.4b2aa660.js b/assets/js/9d5eac8e.6a3edbf9.js
similarity index 98%
rename from assets/js/9d5eac8e.4b2aa660.js
rename to assets/js/9d5eac8e.6a3edbf9.js
index c9e452ba..de88a4c9 100644
--- a/assets/js/9d5eac8e.4b2aa660.js
+++ b/assets/js/9d5eac8e.6a3edbf9.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[93],{163:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(232)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-1.2.11/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-1.2.11/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/documentation_guide.md",version:"1.2.11",sidebar_label:"Docusaurus",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"}},l=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],c={toc:l};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),u=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},c),{},{components:n})):a.a.createElement(h,s({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var c=2;c<i;c++)r[c]=n[c];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[93],{163:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(228)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-1.2.11/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-1.2.11/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/1.2.11/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/documentation_guide.md",version:"1.2.11",sidebar_label:"Docusaurus",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"}},l=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],c={toc:l};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),u=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},c),{},{components:n})):a.a.createElement(h,s({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var c=2;c<i;c++)r[c]=n[c];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9e1ec80c.ab114838.js b/assets/js/9e1ec80c.ab114838.js
deleted file mode 100644
index 9ab221f4..00000000
--- a/assets/js/9e1ec80c.ab114838.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[94],{164:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-2.0.0/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-2.0.0/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/logstash.md",version:"2.0.0",sidebar_label:"Logstash",sidebar:"version-2.0.0/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9f628d52.08c37d03.js b/assets/js/9f628d52.08c37d03.js
new file mode 100644
index 00000000..69a43922
--- /dev/null
+++ b/assets/js/9f628d52.08c37d03.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[94],{164:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-2.1.0/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-2.1.0/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_install_advanced.md",version:"2.1.0",sidebar_label:"Docker Advanced Options",sidebar:"version-2.1.0/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(i,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,l(l({ref:t},c),{},{components:n})):a.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9fe55709.25f4e00b.js b/assets/js/9fe55709.e9c7bfda.js
similarity index 99%
rename from assets/js/9fe55709.25f4e00b.js
rename to assets/js/9fe55709.e9c7bfda.js
index 04512706..b368fcd9 100644
--- a/assets/js/9fe55709.25f4e00b.js
+++ b/assets/js/9fe55709.e9c7bfda.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[95],{165:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return p}));var o=n(3),i=n(7),a=(n(0),n(232)),s={id:"install",title:"NetSage Flow Processing Pipeline Install Guide",sidebar_label:"Installation Guide"},r={unversionedId:"deploy/install",id:"version-1.2.5/deploy/install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Install Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.5/deploy/install.md",slug:"/deploy/install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/install.md",version:"1.2.5",sidebar_label:"Installation Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}},l=[{value:"Components",id:"components",children:[]},{value:"Prerequirements",id:"prerequirements",children:[{value:"nfdump",id:"nfdump",children:[]}]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"RabbitMQ",id:"rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline",id:"installing-the-pipeline",children:[]},{value:"Configuration",id:"configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Configuring the Pipeline Stages",id:"configuring-the-pipeline-stages",children:[]},{value:"Shared config file listing",id:"shared-config-file-listing",children:[]}]},{value:"Running the daemons",id:"running-the-daemons",children:[{value:"Daemon Listing",id:"daemon-listing",children:[]}]},{value:"Setup Notes",id:"setup-notes",children:[]}],c={toc:l};function p(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(a.b)("h2",{id:"components"},"Components"),Object(a.b)("p",null,"Minimum components"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Data Injestion Source (nfdump, tstat or both)"),Object(a.b)("li",{parentName:"ul"},"RabbitMQ"),Object(a.b)("li",{parentName:"ul"},"LogStash"),Object(a.b)("li",{parentName:"ul"},"Importer (If you use nfdump)")),Object(a.b)("h2",{id:"prerequirements"},"Prerequirements"),Object(a.b)("p",null,"The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed."),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"nfdump "),Object(a.b)("li",{parentName:"ol"},"tstat")),Object(a.b)("p",null,"You'll need to have at least one of them setup in order to be able to process the data"),Object(a.b)("h3",{id:"nfdump"},"nfdump"),Object(a.b)("p",null,"The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is ",Object(a.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(a.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(a.b)("p",null,"Once nfdump is setup you'll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. "),Object(a.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(a.b)("h3",{id:"rabbitmq"},"RabbitMQ"),Object(a.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n`\n\n")),Object(a.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service\n")),Object(a.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(a.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(a.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(a.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(a.b)("p",null,"For Red Hat/CentOS 6, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(a.b)("p",null,"For Red Hat/CentOS 7, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(a.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(a.b)("h2",{id:"installing-the-pipeline"},"Installing the Pipeline"),Object(a.b)("p",null,"Pipeline components:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise."),Object(a.b)("li",{parentName:"ol"},"Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them")),Object(a.b)("p",null,"send directly to the appropriate rabbit queue. "),Object(a.b)("ol",{start:3},Object(a.b)("li",{parentName:"ol"},"Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows ")),Object(a.b)("p",null,"that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.\nThe final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the\nlast logstash conf here to write to elasticsearch."),Object(a.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. Install it like this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(a.b)("h2",{id:"configuration"},"Configuration"),Object(a.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(a.b)("p",null,"Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files ","[THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]",". This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")," "),Object(a.b)("p",null,"The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'<collection>\n    <flow-path>/path/to/flow-files</flow-path>\n    <sensor>hostname.tld</sensor>\n\x3c!--  "instance" goes along with sensor\n       This is to identify various instances if a sensor has more than one "stream" / data collection\n       Defaults to 0.\n    <instance>1</instance>\n--\x3e\n  \x3c!--\n       Defaults to sensor, but you can set it to something else here\n      <router-address></router-address>\n  --\x3e\n \x3c!--\n    Flow type: type of flow data (defaults to netflow)\n --\x3e\n \x3c!--\n    <flow-type>sflow</flow-type>\n --\x3e\n</collection>\n')),Object(a.b)("p",null,"Notice that ",Object(a.b)("inlineCode",{parentName:"p"},"instance")," , ",Object(a.b)("inlineCode",{parentName:"p"},"router-address")," , and ",Object(a.b)("inlineCode",{parentName:"p"},"flow-type")," are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file."),Object(a.b)("p",null,"You can have multiple ",Object(a.b)("inlineCode",{parentName:"p"},"collection")," stanzas, to import multiple collections on one host."),Object(a.b)("p",null,"The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"<config>\n  <collection>\n    <flow-path>/path/to/flow-files1</flow-path>\n    <sensor>hostname1.tld</sensor>\n  </collection>\n  <collection>\n    <flow-path>/path/to/flow-files2</flow-path>\n    <sensor>hostname2.tld</sensor>\n  </collection>\n\n  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_input>\n\n  \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_output>\n</config>\n")),Object(a.b)("h3",{id:"configuring-the-pipeline-stages"},"Configuring the Pipeline Stages"),Object(a.b)("p",null,"Each stage must be configured with Rabbit input and output queue information. The intention here is that flows should be deidentified before they leave the original node the flow data is collected on. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(a.b)("p",null,'The username/password are both set to "guest" by default, as this is the default provided by RabbitMQ. This works fine if the localhost is processing all the data. The configs look something like this (some have additional sections).'),Object(a.b)("p",null,"Notice that the only Rabbit connection information that's provided here is that which is not specific in the shared config file. This way if we need to change the credentials throughout the entire pipeline, it's easy to do."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-xml"},"   <config>\n     \x3c!-- rabbitmq connection info --\x3e\n     <rabbit_input>\n       <queue>netsage_deidentifier_raw</queue>\n       <channel>2</channel>\n     </rabbit_input>\n   \n     \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n     <rabbit_output>\n       <queue>netsage_deidentifier_cached</queue>\n       <channel>3</channel>\n     </rabbit_output>\n     <worker>\n         \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n         \x3c!-- for stitching, we can only use 1 --\x3e\n       <num-processes>1</num-processes>\n   \n       \x3c!-- where should we write the cache worker pid file to --\x3e\n       <pid-file>/var/run/netsage-cache-workers.pid</pid-file>\n   \n     </worker>\n   </config>\n")),Object(a.b)("p",null,"The defaults should work unless the pipeline stages need to be reordered for some reason, or if SSL or different hosts/credentials are needed. However, the very endpoints should be checked. At the moment that means the flow cache (which is the first stage in the pipeline) and the flow mover (the last stage)."),Object(a.b)("h3",{id:"shared-config-file-listing"},"Shared config file listing"),Object(a.b)("p",null,"The shared configuration files and logging configuration files are listed below (all of the pipeline components use these):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information\n/etc/grnoc/netsage/deidentifier/logging.conf - logging config\n/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled\n")),Object(a.b)("h2",{id:"running-the-daemons"},"Running the daemons"),Object(a.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(a.b)("p",null,"For more details on each individual daemon, use the ",Object(a.b)("inlineCode",{parentName:"p"},"--help")," flag."),Object(a.b)("h3",{id:"daemon-listing"},"Daemon Listing"),Object(a.b)("h4",{id:"netsage-netflow-importer-daemon"},"netsage-netflow-importer-daemon"),Object(a.b)("p",null,"This is a daemon that reads raw netflow data, reads it, and pushes it to a Rabbit queue for processing."),Object(a.b)("p",null,"Config file: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")," "),Object(a.b)("h2",{id:"setup-notes"},"Setup Notes"),Object(a.b)("p",null,"INPUT AND OUTPUT LOGSTASH FILTERS\nStandard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs (01-inputs.conf and 99-outputs.conf) may be modified for your use.\nTo use the provided 01-inputs.conf and 99-outputs.conf versions, fill in the IP of the final rabbit host in 99-outputs.conf, and put the rabbitmq usernames and passwords into the logstash keystore.\nYour 01 and 99 conf files should not be overwritten by upgrades."),Object(a.b)("p",null,"To set up the keystore:  (note that logstash-keystore takes a minute to come back with a prompt)\nBe sure /usr/share/logstash/config exists\n(the full path, in case you need it: /usr/share/logstash/bin/logstash-keystore)\nCreate logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create\n    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)\n```sh\nTo list the keys:\n")),Object(a.b)("p",null,"  $ sudo -E logstash-keystore list"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"To remove a key-value pair:\n\n```sh\n  $ sudo -E logstash-keystore remove <key name>\n")),Object(a.b)("p",null,'FLOW STITCHING - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use "-w 1".'),Object(a.b)("p",null,"See the comments in 04-stitching.conf to learn more about how complete flows are defined."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),i=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,i=function(e,t){if(null==e)return{};var n,o,i={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),p=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},h=i.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(s,".").concat(h)]||u[h]||d[h]||a;return n?i.a.createElement(b,r(r({ref:t},c),{},{components:n})):i.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:o,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return i.a.createElement.apply(null,s)}return i.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[95],{165:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return p}));var o=n(3),i=n(7),a=(n(0),n(228)),s={id:"install",title:"NetSage Flow Processing Pipeline Install Guide",sidebar_label:"Installation Guide"},r={unversionedId:"deploy/install",id:"version-1.2.5/deploy/install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Install Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.5/deploy/install.md",slug:"/deploy/install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/install.md",version:"1.2.5",sidebar_label:"Installation Guide",sidebar:"version-1.2.5/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install"}},l=[{value:"Components",id:"components",children:[]},{value:"Prerequirements",id:"prerequirements",children:[{value:"nfdump",id:"nfdump",children:[]}]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"RabbitMQ",id:"rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline",id:"installing-the-pipeline",children:[]},{value:"Configuration",id:"configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Configuring the Pipeline Stages",id:"configuring-the-pipeline-stages",children:[]},{value:"Shared config file listing",id:"shared-config-file-listing",children:[]}]},{value:"Running the daemons",id:"running-the-daemons",children:[{value:"Daemon Listing",id:"daemon-listing",children:[]}]},{value:"Setup Notes",id:"setup-notes",children:[]}],c={toc:l};function p(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(a.b)("h2",{id:"components"},"Components"),Object(a.b)("p",null,"Minimum components"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Data Injestion Source (nfdump, tstat or both)"),Object(a.b)("li",{parentName:"ul"},"RabbitMQ"),Object(a.b)("li",{parentName:"ul"},"LogStash"),Object(a.b)("li",{parentName:"ul"},"Importer (If you use nfdump)")),Object(a.b)("h2",{id:"prerequirements"},"Prerequirements"),Object(a.b)("p",null,"The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed."),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"nfdump "),Object(a.b)("li",{parentName:"ol"},"tstat")),Object(a.b)("p",null,"You'll need to have at least one of them setup in order to be able to process the data"),Object(a.b)("h3",{id:"nfdump"},"nfdump"),Object(a.b)("p",null,"The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is ",Object(a.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(a.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(a.b)("p",null,"Once nfdump is setup you'll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. "),Object(a.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(a.b)("h3",{id:"rabbitmq"},"RabbitMQ"),Object(a.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n`\n\n")),Object(a.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service\n")),Object(a.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(a.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(a.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(a.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(a.b)("p",null,"For Red Hat/CentOS 6, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(a.b)("p",null,"For Red Hat/CentOS 7, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(a.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(a.b)("h2",{id:"installing-the-pipeline"},"Installing the Pipeline"),Object(a.b)("p",null,"Pipeline components:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise."),Object(a.b)("li",{parentName:"ol"},"Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them")),Object(a.b)("p",null,"send directly to the appropriate rabbit queue. "),Object(a.b)("ol",{start:3},Object(a.b)("li",{parentName:"ol"},"Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows ")),Object(a.b)("p",null,"that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.\nThe final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the\nlast logstash conf here to write to elasticsearch."),Object(a.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. Install it like this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(a.b)("h2",{id:"configuration"},"Configuration"),Object(a.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(a.b)("p",null,"Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files ","[THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]",". This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")," "),Object(a.b)("p",null,"The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'<collection>\n    <flow-path>/path/to/flow-files</flow-path>\n    <sensor>hostname.tld</sensor>\n\x3c!--  "instance" goes along with sensor\n       This is to identify various instances if a sensor has more than one "stream" / data collection\n       Defaults to 0.\n    <instance>1</instance>\n--\x3e\n  \x3c!--\n       Defaults to sensor, but you can set it to something else here\n      <router-address></router-address>\n  --\x3e\n \x3c!--\n    Flow type: type of flow data (defaults to netflow)\n --\x3e\n \x3c!--\n    <flow-type>sflow</flow-type>\n --\x3e\n</collection>\n')),Object(a.b)("p",null,"Notice that ",Object(a.b)("inlineCode",{parentName:"p"},"instance")," , ",Object(a.b)("inlineCode",{parentName:"p"},"router-address")," , and ",Object(a.b)("inlineCode",{parentName:"p"},"flow-type")," are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file."),Object(a.b)("p",null,"You can have multiple ",Object(a.b)("inlineCode",{parentName:"p"},"collection")," stanzas, to import multiple collections on one host."),Object(a.b)("p",null,"The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"<config>\n  <collection>\n    <flow-path>/path/to/flow-files1</flow-path>\n    <sensor>hostname1.tld</sensor>\n  </collection>\n  <collection>\n    <flow-path>/path/to/flow-files2</flow-path>\n    <sensor>hostname2.tld</sensor>\n  </collection>\n\n  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_input>\n\n  \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_output>\n</config>\n")),Object(a.b)("h3",{id:"configuring-the-pipeline-stages"},"Configuring the Pipeline Stages"),Object(a.b)("p",null,"Each stage must be configured with Rabbit input and output queue information. The intention here is that flows should be deidentified before they leave the original node the flow data is collected on. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(a.b)("p",null,'The username/password are both set to "guest" by default, as this is the default provided by RabbitMQ. This works fine if the localhost is processing all the data. The configs look something like this (some have additional sections).'),Object(a.b)("p",null,"Notice that the only Rabbit connection information that's provided here is that which is not specific in the shared config file. This way if we need to change the credentials throughout the entire pipeline, it's easy to do."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-xml"},"   <config>\n     \x3c!-- rabbitmq connection info --\x3e\n     <rabbit_input>\n       <queue>netsage_deidentifier_raw</queue>\n       <channel>2</channel>\n     </rabbit_input>\n   \n     \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n     <rabbit_output>\n       <queue>netsage_deidentifier_cached</queue>\n       <channel>3</channel>\n     </rabbit_output>\n     <worker>\n         \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n         \x3c!-- for stitching, we can only use 1 --\x3e\n       <num-processes>1</num-processes>\n   \n       \x3c!-- where should we write the cache worker pid file to --\x3e\n       <pid-file>/var/run/netsage-cache-workers.pid</pid-file>\n   \n     </worker>\n   </config>\n")),Object(a.b)("p",null,"The defaults should work unless the pipeline stages need to be reordered for some reason, or if SSL or different hosts/credentials are needed. However, the very endpoints should be checked. At the moment that means the flow cache (which is the first stage in the pipeline) and the flow mover (the last stage)."),Object(a.b)("h3",{id:"shared-config-file-listing"},"Shared config file listing"),Object(a.b)("p",null,"The shared configuration files and logging configuration files are listed below (all of the pipeline components use these):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information\n/etc/grnoc/netsage/deidentifier/logging.conf - logging config\n/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled\n")),Object(a.b)("h2",{id:"running-the-daemons"},"Running the daemons"),Object(a.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(a.b)("p",null,"For more details on each individual daemon, use the ",Object(a.b)("inlineCode",{parentName:"p"},"--help")," flag."),Object(a.b)("h3",{id:"daemon-listing"},"Daemon Listing"),Object(a.b)("h4",{id:"netsage-netflow-importer-daemon"},"netsage-netflow-importer-daemon"),Object(a.b)("p",null,"This is a daemon that reads raw netflow data, reads it, and pushes it to a Rabbit queue for processing."),Object(a.b)("p",null,"Config file: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")," "),Object(a.b)("h2",{id:"setup-notes"},"Setup Notes"),Object(a.b)("p",null,"INPUT AND OUTPUT LOGSTASH FILTERS\nStandard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs (01-inputs.conf and 99-outputs.conf) may be modified for your use.\nTo use the provided 01-inputs.conf and 99-outputs.conf versions, fill in the IP of the final rabbit host in 99-outputs.conf, and put the rabbitmq usernames and passwords into the logstash keystore.\nYour 01 and 99 conf files should not be overwritten by upgrades."),Object(a.b)("p",null,"To set up the keystore:  (note that logstash-keystore takes a minute to come back with a prompt)\nBe sure /usr/share/logstash/config exists\n(the full path, in case you need it: /usr/share/logstash/bin/logstash-keystore)\nCreate logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create\n    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)\n```sh\nTo list the keys:\n")),Object(a.b)("p",null,"  $ sudo -E logstash-keystore list"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"To remove a key-value pair:\n\n```sh\n  $ sudo -E logstash-keystore remove <key name>\n")),Object(a.b)("p",null,'FLOW STITCHING - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use "-w 1".'),Object(a.b)("p",null,"See the comments in 04-stitching.conf to learn more about how complete flows are defined."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),i=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,i=function(e,t){if(null==e)return{};var n,o,i={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),p=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},h=i.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(s,".").concat(h)]||u[h]||d[h]||a;return n?i.a.createElement(b,r(r({ref:t},c),{},{components:n})):i.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:o,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return i.a.createElement.apply(null,s)}return i.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/9feaf8ab.31cb6a73.js b/assets/js/9feaf8ab.8c912915.js
similarity index 99%
rename from assets/js/9feaf8ab.31cb6a73.js
rename to assets/js/9feaf8ab.8c912915.js
index 5db20d94..6c674da8 100644
--- a/assets/js/9feaf8ab.31cb6a73.js
+++ b/assets/js/9feaf8ab.8c912915.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[96],{166:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),r=(n(0),n(232)),l={id:"docker_install",title:"Docker Installation Guide",sidebar_label:"Docker Install"},i={unversionedId:"deploy/docker_install",id:"version-1.2.6/deploy/docker_install",isDocsHomePage:!1,title:"Docker Installation Guide",description:"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all",source:"@site/versioned_docs/version-1.2.6/deploy/docker_install.md",slug:"/deploy/docker_install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/docker_install.md",version:"1.2.6",sidebar_label:"Docker Install",sidebar:"version-1.2.6/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.6/docusaurus"}},c=[{value:"Nfdump",id:"nfdump",children:[{value:"External Nfdump",id:"external-nfdump",children:[]},{value:"Dockerized Nfdump",id:"dockerized-nfdump",children:[]}]},{value:"Common Pattern",id:"common-pattern",children:[{value:"Environment file",id:"environment-file",children:[]},{value:"Bringing up the Pipeline",id:"bringing-up-the-pipeline",children:[]},{value:"Shutting Down the pipeline.",id:"shutting-down-the-pipeline",children:[]},{value:"Advanced Configuration",id:"advanced-configuration",children:[]},{value:"Kibana and Elastic Search",id:"kibana-and-elastic-search",children:[]}]},{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]},{value:"Upgrading",id:"upgrading",children:[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}]}],s={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all\nthe nuances of getting the pipeline setup."),Object(r.b)("p",null,"Before we start, you may have a read over the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"developer docker guide")," it contains several notes such as how to select the docker version and likely other bits of information you may find useful."),Object(r.b)("h2",{id:"nfdump"},"Nfdump"),Object(r.b)("p",null,"Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you'll be defining."),Object(r.b)("p",null,"More info of nfdump can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump/"},"here")),Object(r.b)("h3",{id:"external-nfdump"},"External Nfdump"),Object(r.b)("p",null,"In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided."),Object(r.b)("p",null,"You'll need to update your scripts to output to \\$PROJECT/data/input_data. Naturally all the paths are configurable but you'll have a much easier if you stick to the defaults."),Object(r.b)("p",null,"If you do choose to store the data elsewhere, the location may still need to be inside of the \\$PROJECT or a docker volume location in order for docker to be able to reference it."),Object(r.b)("p",null,"You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data."),Object(r.b)("p",null,"At this point please proceed to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h3",{id:"dockerized-nfdump"},"Dockerized Nfdump"),Object(r.b)("p",null,"After you've selected the version of docker you'll be running. you can start the collector by simply running:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d collector\n")),Object(r.b)("p",null,"The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."),Object(r.b)("p",null,"By default the container comes up and will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"./data/input_data")," and listen to udp traffic on localhost:9999."),Object(r.b)("p",null,"continue to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h2",{id:"common-pattern"},"Common Pattern"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable or using the 'Release' version."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Development Version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},"Disregard anything about ",Object(r.b)("inlineCode",{parentName:"li"},"docker_select_version.sh")," that will not apply to you "),Object(r.b)("li",{parentName:"ul"},"Update to latest code git pull origin master"))),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Release version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("inlineCode",{parentName:"li"},"git fetch; git checkout <tag name>"),' replace "tag name" with v1.2.5 or the version you intend to use.'),Object(r.b)("li",{parentName:"ul"},"Please select the version you wish to use using ",Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh"),"\nit is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.")))),Object(r.b)("h3",{id:"environment-file"},"Environment file"),Object(r.b)("p",null,"Please make a copy of the .env and refer back to the docker ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"dev guide")," on details on configuring the env. Most of the default value should work just fine."),Object(r.b)("p",null,"The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as."),Object(r.b)("p",null,"If you don't send a sensor name it'll use the default docker hostname which changes each time you run the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n\nsflowSensorName=sflowSensorName\nnetflowSensorName=netflowSensorName\n\n")),Object(r.b)("p",null,"Please note, the default is to have one netflow collector and one sflow collector.  If you need more collectors or do no need netflow or sflow simply comment out the collector you wish to ignore."),Object(r.b)("h3",{id:"bringing-up-the-pipeline"},"Bringing up the Pipeline"),Object(r.b)("p",null,"Starting up the pipeline using:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",null,"You can check the logs for each of the container by running"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("h3",{id:"shutting-down-the-pipeline"},"Shutting Down the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")),Object(r.b)("h3",{id:"advanced-configuration"},"Advanced Configuration"),Object(r.b)("p",null,"The pipeline allows to have as many collectors as desired.  You should have a unique sensorName ENV variable for each type and a unique path where data is being delivered."),Object(r.b)("p",null,"By convention everything is being written to ./data/input_data/sensorName You may change that behavior but just ensure the path between the colle"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"Copy the compose/importer/netsage_shared.xml to userConfig/ and name it netsage_override.xml"),Object(r.b)("li",{parentName:"ol"},"In the docker-compose.yml uncomment the following line from the importer configuration.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"\n      - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n\n")),Object(r.b)("p",null,"This will use the ",Object(r.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," in the userConfig instead of the container settings."),Object(r.b)("ol",{start:3},Object(r.b)("li",{parentName:"ol"},"Update collectors.")),Object(r.b)("p",null,"You may add as many new collectors as you like just ensure the following is unique:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    command: nfcapd -T all -l /data -S 1 -w -z -p 9999\n    ports:\n\n      - "9999:9999/udp"\n\n    restart: always\n    volumes:\n\n      - ./data/input_data/example:/data\n\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"The command call should be updated.  nfcapd for netflow, sfcapd for sflow")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"The output under volumes needs to be unique. Replace /example with the appropriate value")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Make sure to update the port.  The UDP port has to be unique.  Please update the command and port mapping.  "),Object(r.b)("p",{parentName:"li"},"Technically you don't need to change to port of the command, but make sure you use the correct pattern when mapping the new settings."))),Object(r.b)("p",null,"Example: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yml"},'ports:\n\n      - "9999:4321/udp"\n\n')),Object(r.b)("p",null,"The first port is the port on your host, the second port is the port on your local machine. "),Object(r.b)("ol",{start:4},Object(r.b)("li",{parentName:"ol"},"Update the netsage_override.xml and add a new entry for the collector you're adding under the config section.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example</flow-path>\n        <sensor>$exampleSensorName</sensor> \n        <flow-type>sflow</flow-type>\n    </collection>\n\n")),Object(r.b)("ol",{start:5},Object(r.b)("li",{parentName:"ol"},"Update the environment file.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=example\n")),Object(r.b)("ol",{start:6},Object(r.b)("li",{parentName:"ol"},"At this point, please update the router configuration to send data to the new port you've defined.  If the new collector is listening on 0.0.0.0:1234/udp then all traffic you wish grouped under ")),Object(r.b)("p",null,"the new sensor should be send to 1234/udp.  "),Object(r.b)("p",null,"You will need to repeat steps 3-6 for each collector you're adding.  For each new configuration the path, sensorName and exposed port have to be unique.  Besides that, there is no limit\noutside of the bounds of the host's resources to how many collectors you may run."),Object(r.b)("h3",{id:"kibana-and-elastic-search"},"Kibana and Elastic Search"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(r.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(r.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(r.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data.\xa0"),Object(r.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(r.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(r.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory."),Object(r.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"environment:\n\n  + LS_JAVA_OPTS=-Xmx3g\n\n")),Object(r.b)("p",null,"Alternatively you may also try doing this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(r.b)("p",null,"Reference: ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(r.b)("p",null,"If your only changes are the version you selected simply reset and discard your changes."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git reset --hard\n")),Object(r.b)("h4",{id:"development"},"Development"),Object(r.b)("p",null,"Update the git repo. Likely this won't change anything but it's always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git pull origin master\n")),Object(r.b)("h4",{id:"release"},"Release"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"git checkout <tag_value> (ie. v1.2.6, v1.2.7 etc)"),Object(r.b)("li",{parentName:"ol"},Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh")," select the same version as the tag you checked out.")),Object(r.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(r.b)("p",null,"This applies for both development and release"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),u=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=u(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),p=u(n),b=o,h=p["".concat(l,".").concat(b)]||p[b]||d[b]||r;return n?a.a.createElement(h,i(i({ref:t},s),{},{components:n})):a.a.createElement(h,i({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,l=new Array(r);l[0]=b;var i={};for(var c in t)hasOwnProperty.call(t,c)&&(i[c]=t[c]);i.originalType=e,i.mdxType="string"==typeof e?e:o,l[1]=i;for(var s=2;s<r;s++)l[s]=n[s];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[96],{166:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),r=(n(0),n(228)),l={id:"docker_install",title:"Docker Installation Guide",sidebar_label:"Docker Install"},i={unversionedId:"deploy/docker_install",id:"version-1.2.6/deploy/docker_install",isDocsHomePage:!1,title:"Docker Installation Guide",description:"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all",source:"@site/versioned_docs/version-1.2.6/deploy/docker_install.md",slug:"/deploy/docker_install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/docker_install.md",version:"1.2.6",sidebar_label:"Docker Install",sidebar:"version-1.2.6/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.6/docusaurus"}},c=[{value:"Nfdump",id:"nfdump",children:[{value:"External Nfdump",id:"external-nfdump",children:[]},{value:"Dockerized Nfdump",id:"dockerized-nfdump",children:[]}]},{value:"Common Pattern",id:"common-pattern",children:[{value:"Environment file",id:"environment-file",children:[]},{value:"Bringing up the Pipeline",id:"bringing-up-the-pipeline",children:[]},{value:"Shutting Down the pipeline.",id:"shutting-down-the-pipeline",children:[]},{value:"Advanced Configuration",id:"advanced-configuration",children:[]},{value:"Kibana and Elastic Search",id:"kibana-and-elastic-search",children:[]}]},{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]},{value:"Upgrading",id:"upgrading",children:[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}]}],s={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all\nthe nuances of getting the pipeline setup."),Object(r.b)("p",null,"Before we start, you may have a read over the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"developer docker guide")," it contains several notes such as how to select the docker version and likely other bits of information you may find useful."),Object(r.b)("h2",{id:"nfdump"},"Nfdump"),Object(r.b)("p",null,"Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you'll be defining."),Object(r.b)("p",null,"More info of nfdump can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump/"},"here")),Object(r.b)("h3",{id:"external-nfdump"},"External Nfdump"),Object(r.b)("p",null,"In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided."),Object(r.b)("p",null,"You'll need to update your scripts to output to \\$PROJECT/data/input_data. Naturally all the paths are configurable but you'll have a much easier if you stick to the defaults."),Object(r.b)("p",null,"If you do choose to store the data elsewhere, the location may still need to be inside of the \\$PROJECT or a docker volume location in order for docker to be able to reference it."),Object(r.b)("p",null,"You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data."),Object(r.b)("p",null,"At this point please proceed to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h3",{id:"dockerized-nfdump"},"Dockerized Nfdump"),Object(r.b)("p",null,"After you've selected the version of docker you'll be running. you can start the collector by simply running:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d collector\n")),Object(r.b)("p",null,"The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."),Object(r.b)("p",null,"By default the container comes up and will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"./data/input_data")," and listen to udp traffic on localhost:9999."),Object(r.b)("p",null,"continue to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h2",{id:"common-pattern"},"Common Pattern"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable or using the 'Release' version."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Development Version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},"Disregard anything about ",Object(r.b)("inlineCode",{parentName:"li"},"docker_select_version.sh")," that will not apply to you "),Object(r.b)("li",{parentName:"ul"},"Update to latest code git pull origin master"))),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Release version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("inlineCode",{parentName:"li"},"git fetch; git checkout <tag name>"),' replace "tag name" with v1.2.5 or the version you intend to use.'),Object(r.b)("li",{parentName:"ul"},"Please select the version you wish to use using ",Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh"),"\nit is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.")))),Object(r.b)("h3",{id:"environment-file"},"Environment file"),Object(r.b)("p",null,"Please make a copy of the .env and refer back to the docker ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"dev guide")," on details on configuring the env. Most of the default value should work just fine."),Object(r.b)("p",null,"The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as."),Object(r.b)("p",null,"If you don't send a sensor name it'll use the default docker hostname which changes each time you run the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n\nsflowSensorName=sflowSensorName\nnetflowSensorName=netflowSensorName\n\n")),Object(r.b)("p",null,"Please note, the default is to have one netflow collector and one sflow collector.  If you need more collectors or do no need netflow or sflow simply comment out the collector you wish to ignore."),Object(r.b)("h3",{id:"bringing-up-the-pipeline"},"Bringing up the Pipeline"),Object(r.b)("p",null,"Starting up the pipeline using:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",null,"You can check the logs for each of the container by running"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("h3",{id:"shutting-down-the-pipeline"},"Shutting Down the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")),Object(r.b)("h3",{id:"advanced-configuration"},"Advanced Configuration"),Object(r.b)("p",null,"The pipeline allows to have as many collectors as desired.  You should have a unique sensorName ENV variable for each type and a unique path where data is being delivered."),Object(r.b)("p",null,"By convention everything is being written to ./data/input_data/sensorName You may change that behavior but just ensure the path between the colle"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"Copy the compose/importer/netsage_shared.xml to userConfig/ and name it netsage_override.xml"),Object(r.b)("li",{parentName:"ol"},"In the docker-compose.yml uncomment the following line from the importer configuration.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"\n      - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n\n")),Object(r.b)("p",null,"This will use the ",Object(r.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," in the userConfig instead of the container settings."),Object(r.b)("ol",{start:3},Object(r.b)("li",{parentName:"ol"},"Update collectors.")),Object(r.b)("p",null,"You may add as many new collectors as you like just ensure the following is unique:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    command: nfcapd -T all -l /data -S 1 -w -z -p 9999\n    ports:\n\n      - "9999:9999/udp"\n\n    restart: always\n    volumes:\n\n      - ./data/input_data/example:/data\n\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"The command call should be updated.  nfcapd for netflow, sfcapd for sflow")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"The output under volumes needs to be unique. Replace /example with the appropriate value")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Make sure to update the port.  The UDP port has to be unique.  Please update the command and port mapping.  "),Object(r.b)("p",{parentName:"li"},"Technically you don't need to change to port of the command, but make sure you use the correct pattern when mapping the new settings."))),Object(r.b)("p",null,"Example: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yml"},'ports:\n\n      - "9999:4321/udp"\n\n')),Object(r.b)("p",null,"The first port is the port on your host, the second port is the port on your local machine. "),Object(r.b)("ol",{start:4},Object(r.b)("li",{parentName:"ol"},"Update the netsage_override.xml and add a new entry for the collector you're adding under the config section.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example</flow-path>\n        <sensor>$exampleSensorName</sensor> \n        <flow-type>sflow</flow-type>\n    </collection>\n\n")),Object(r.b)("ol",{start:5},Object(r.b)("li",{parentName:"ol"},"Update the environment file.")),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=example\n")),Object(r.b)("ol",{start:6},Object(r.b)("li",{parentName:"ol"},"At this point, please update the router configuration to send data to the new port you've defined.  If the new collector is listening on 0.0.0.0:1234/udp then all traffic you wish grouped under ")),Object(r.b)("p",null,"the new sensor should be send to 1234/udp.  "),Object(r.b)("p",null,"You will need to repeat steps 3-6 for each collector you're adding.  For each new configuration the path, sensorName and exposed port have to be unique.  Besides that, there is no limit\noutside of the bounds of the host's resources to how many collectors you may run."),Object(r.b)("h3",{id:"kibana-and-elastic-search"},"Kibana and Elastic Search"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(r.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(r.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(r.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data.\xa0"),Object(r.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(r.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(r.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory."),Object(r.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"environment:\n\n  + LS_JAVA_OPTS=-Xmx3g\n\n")),Object(r.b)("p",null,"Alternatively you may also try doing this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(r.b)("p",null,"Reference: ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(r.b)("p",null,"If your only changes are the version you selected simply reset and discard your changes."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git reset --hard\n")),Object(r.b)("h4",{id:"development"},"Development"),Object(r.b)("p",null,"Update the git repo. Likely this won't change anything but it's always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git pull origin master\n")),Object(r.b)("h4",{id:"release"},"Release"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"git checkout <tag_value> (ie. v1.2.6, v1.2.7 etc)"),Object(r.b)("li",{parentName:"ol"},Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh")," select the same version as the tag you checked out.")),Object(r.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(r.b)("p",null,"This applies for both development and release"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),u=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=u(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),p=u(n),b=o,h=p["".concat(l,".").concat(b)]||p[b]||d[b]||r;return n?a.a.createElement(h,i(i({ref:t},s),{},{components:n})):a.a.createElement(h,i({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,l=new Array(r);l[0]=b;var i={};for(var c in t)hasOwnProperty.call(t,c)&&(i[c]=t[c]);i.originalType=e,i.mdxType="string"==typeof e?e:o,l[1]=i;for(var s=2;s<r;s++)l[s]=n[s];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a04bc459.cf90e3e8.js b/assets/js/a04bc459.dcff4f83.js
similarity index 98%
rename from assets/js/a04bc459.cf90e3e8.js
rename to assets/js/a04bc459.dcff4f83.js
index d9b8c5f2..905ab51e 100644
--- a/assets/js/a04bc459.cf90e3e8.js
+++ b/assets/js/a04bc459.dcff4f83.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[97],{167:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install",title:"Docker Installation Guide",sidebar_label:"Docker Install"},l={unversionedId:"deploy/docker_install",id:"version-1.2.5/deploy/docker_install",isDocsHomePage:!1,title:"Docker Installation Guide",description:"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all",source:"@site/versioned_docs/version-1.2.5/deploy/docker_install.md",slug:"/deploy/docker_install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/docker_install.md",version:"1.2.5",sidebar_label:"Docker Install",sidebar:"version-1.2.5/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.5/docusaurus"}},c=[{value:"Nfdump",id:"nfdump",children:[{value:"External Nfdump",id:"external-nfdump",children:[]},{value:"Dockerized Nfdump",id:"dockerized-nfdump",children:[]}]},{value:"Common Pattern",id:"common-pattern",children:[{value:"Environment file",id:"environment-file",children:[]},{value:"Bringing up the Pipeline",id:"bringing-up-the-pipeline",children:[]},{value:"Shutting Down the pipeline.",id:"shutting-down-the-pipeline",children:[]},{value:"Kibana and Elastic Search",id:"kibana-and-elastic-search",children:[]}]},{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]},{value:"Upgrading",id:"upgrading",children:[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}]}],s={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all\nthe nuances of getting the pipeline setup."),Object(r.b)("p",null,"Before we start, you may have a read over the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"developer docker guide")," it contains several notes such as how to select the docker version and likely other bits of information you may find useful."),Object(r.b)("h2",{id:"nfdump"},"Nfdump"),Object(r.b)("p",null,"Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you'll be defining."),Object(r.b)("p",null,"More info of nfdump can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump/"},"here")),Object(r.b)("h3",{id:"external-nfdump"},"External Nfdump"),Object(r.b)("p",null,"In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided."),Object(r.b)("p",null,"You'll need to update your scripts to output to \\$PROJECT/data/input_data. Naturally all the paths are configurable but you'll have a much easier if you stick to the defaults."),Object(r.b)("p",null,"If you do choose to store the data elsewhere, the location may still need to be inside of the \\$PROJECT or a docker volume location in order for docker to be able to reference it."),Object(r.b)("p",null,"You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data."),Object(r.b)("p",null,"At this point please proceed to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h3",{id:"dockerized-nfdump"},"Dockerized Nfdump"),Object(r.b)("p",null,"After you've selected the version of docker you'll be running. you can start the collector by simply running:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d collector\n")),Object(r.b)("p",null,"The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."),Object(r.b)("p",null,"By default the container comes up and will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"./data/input_data")," and listen to udp traffic on localhost:9999."),Object(r.b)("p",null,"continue to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h2",{id:"common-pattern"},"Common Pattern"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable or using the 'Release' version."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Development Version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},"Disregard anything about ",Object(r.b)("inlineCode",{parentName:"li"},"docker_select_version.sh")," that will not apply to you "),Object(r.b)("li",{parentName:"ul"},"Update to latest code git pull origin master"))),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Release version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("inlineCode",{parentName:"li"},"git fetch; git checkout <tag name>"),' replace "tag name" with v1.2.5 or the version you intend to use.'),Object(r.b)("li",{parentName:"ul"},"Please select the version you wish to use using ",Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh"),"\nit is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.")))),Object(r.b)("h3",{id:"environment-file"},"Environment file"),Object(r.b)("p",null,"Please make a copy of the .env and refer back to the docker ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"dev guide")," on details on configuring the env. Most of the default value should work just fine."),Object(r.b)("p",null,"The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as."),Object(r.b)("p",null,"If you don't send a sensor name it'll use the default docker hostname which changes each time you run the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n\nsensorName=bestSensorEver\n\n")),Object(r.b)("h3",{id:"bringing-up-the-pipeline"},"Bringing up the Pipeline"),Object(r.b)("p",null,"Starting up the pipeline using:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",null,"You can check the logs for each of the container by running"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("h3",{id:"shutting-down-the-pipeline"},"Shutting Down the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")),Object(r.b)("h3",{id:"kibana-and-elastic-search"},"Kibana and Elastic Search"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(r.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(r.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(r.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data.\xa0"),Object(r.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(r.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(r.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory."),Object(r.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"environment:\n\n  + LS_JAVA_OPTS=-Xmx3g\n\n")),Object(r.b)("p",null,"Alternatively you may also try doing this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(r.b)("p",null,"Reference: ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(r.b)("p",null,"If your only changes are the version you selected simply reset and discard your changes."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git reset --hard\n")),Object(r.b)("h4",{id:"development"},"Development"),Object(r.b)("p",null,"Update the git repo. Likely this won't change anything but it's always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git pull origin master\n")),Object(r.b)("h4",{id:"release"},"Release"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"git checkout <tag_value> (ie. v1.2.6, v1.2.7 etc)"),Object(r.b)("li",{parentName:"ol"},Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh")," select the same version as the tag you checked out.")),Object(r.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(r.b)("p",null,"This applies for both development and release"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),u=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),p=u(n),b=o,h=p["".concat(i,".").concat(b)]||p[b]||d[b]||r;return n?a.a.createElement(h,l(l({ref:t},s),{},{components:n})):a.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<r;s++)i[s]=n[s];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[97],{167:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install",title:"Docker Installation Guide",sidebar_label:"Docker Install"},l={unversionedId:"deploy/docker_install",id:"version-1.2.5/deploy/docker_install",isDocsHomePage:!1,title:"Docker Installation Guide",description:"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all",source:"@site/versioned_docs/version-1.2.5/deploy/docker_install.md",slug:"/deploy/docker_install",permalink:"/netsage-pipeline/docs/1.2.5/deploy/docker_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/docker_install.md",version:"1.2.5",sidebar_label:"Docker Install",sidebar:"version-1.2.5/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.5/deploy/install"},next:{title:"Documentation Guide",permalink:"/netsage-pipeline/docs/1.2.5/docusaurus"}},c=[{value:"Nfdump",id:"nfdump",children:[{value:"External Nfdump",id:"external-nfdump",children:[]},{value:"Dockerized Nfdump",id:"dockerized-nfdump",children:[]}]},{value:"Common Pattern",id:"common-pattern",children:[{value:"Environment file",id:"environment-file",children:[]},{value:"Bringing up the Pipeline",id:"bringing-up-the-pipeline",children:[]},{value:"Shutting Down the pipeline.",id:"shutting-down-the-pipeline",children:[]},{value:"Kibana and Elastic Search",id:"kibana-and-elastic-search",children:[]}]},{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]},{value:"Upgrading",id:"upgrading",children:[{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Update docker containers",id:"update-docker-containers",children:[]}]}],s={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all\nthe nuances of getting the pipeline setup."),Object(r.b)("p",null,"Before we start, you may have a read over the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"developer docker guide")," it contains several notes such as how to select the docker version and likely other bits of information you may find useful."),Object(r.b)("h2",{id:"nfdump"},"Nfdump"),Object(r.b)("p",null,"Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you'll be defining."),Object(r.b)("p",null,"More info of nfdump can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump/"},"here")),Object(r.b)("h3",{id:"external-nfdump"},"External Nfdump"),Object(r.b)("p",null,"In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided."),Object(r.b)("p",null,"You'll need to update your scripts to output to \\$PROJECT/data/input_data. Naturally all the paths are configurable but you'll have a much easier if you stick to the defaults."),Object(r.b)("p",null,"If you do choose to store the data elsewhere, the location may still need to be inside of the \\$PROJECT or a docker volume location in order for docker to be able to reference it."),Object(r.b)("p",null,"You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data."),Object(r.b)("p",null,"At this point please proceed to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h3",{id:"dockerized-nfdump"},"Dockerized Nfdump"),Object(r.b)("p",null,"After you've selected the version of docker you'll be running. you can start the collector by simply running:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d collector\n")),Object(r.b)("p",null,"The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."),Object(r.b)("p",null,"By default the container comes up and will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"./data/input_data")," and listen to udp traffic on localhost:9999."),Object(r.b)("p",null,"continue to ",Object(r.b)("a",{parentName:"p",href:"#common-pattern"},"Common Pattern")),Object(r.b)("h2",{id:"common-pattern"},"Common Pattern"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable or using the 'Release' version."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Development Version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},"Disregard anything about ",Object(r.b)("inlineCode",{parentName:"li"},"docker_select_version.sh")," that will not apply to you "),Object(r.b)("li",{parentName:"ul"},"Update to latest code git pull origin master"))),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Release version"),Object(r.b)("ul",{parentName:"li"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("inlineCode",{parentName:"li"},"git fetch; git checkout <tag name>"),' replace "tag name" with v1.2.5 or the version you intend to use.'),Object(r.b)("li",{parentName:"ul"},"Please select the version you wish to use using ",Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh"),"\nit is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.")))),Object(r.b)("h3",{id:"environment-file"},"Environment file"),Object(r.b)("p",null,"Please make a copy of the .env and refer back to the docker ",Object(r.b)("a",{parentName:"p",href:"../devel/docker"},"dev guide")," on details on configuring the env. Most of the default value should work just fine."),Object(r.b)("p",null,"The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as."),Object(r.b)("p",null,"If you don't send a sensor name it'll use the default docker hostname which changes each time you run the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n\nsensorName=bestSensorEver\n\n")),Object(r.b)("h3",{id:"bringing-up-the-pipeline"},"Bringing up the Pipeline"),Object(r.b)("p",null,"Starting up the pipeline using:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",null,"You can check the logs for each of the container by running"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("h3",{id:"shutting-down-the-pipeline"},"Shutting Down the pipeline."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")),Object(r.b)("h3",{id:"kibana-and-elastic-search"},"Kibana and Elastic Search"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(r.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(r.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(r.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data.\xa0"),Object(r.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(r.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(r.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory."),Object(r.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"environment:\n\n  + LS_JAVA_OPTS=-Xmx3g\n\n")),Object(r.b)("p",null,"Alternatively you may also try doing this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(r.b)("p",null,"Reference: ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(r.b)("p",null,"If your only changes are the version you selected simply reset and discard your changes."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git reset --hard\n")),Object(r.b)("h4",{id:"development"},"Development"),Object(r.b)("p",null,"Update the git repo. Likely this won't change anything but it's always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"git pull origin master\n")),Object(r.b)("h4",{id:"release"},"Release"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"git checkout <tag_value> (ie. v1.2.6, v1.2.7 etc)"),Object(r.b)("li",{parentName:"ol"},Object(r.b)("inlineCode",{parentName:"li"},"./scripts/docker_select_version.sh")," select the same version as the tag you checked out.")),Object(r.b)("h3",{id:"update-docker-containers"},"Update docker containers"),Object(r.b)("p",null,"This applies for both development and release"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=a.a.createContext({}),u=function(e){var t=a.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return a.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),p=u(n),b=o,h=p["".concat(i,".").concat(b)]||p[b]||d[b]||r;return n?a.a.createElement(h,l(l({ref:t},s),{},{components:n})):a.a.createElement(h,l({ref:t},s))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s<r;s++)i[s]=n[s];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a12215ea.35e2776f.js b/assets/js/a12215ea.1ea3e4ef.js
similarity index 98%
rename from assets/js/a12215ea.35e2776f.js
rename to assets/js/a12215ea.1ea3e4ef.js
index d00d7cb8..eec2adec 100644
--- a/assets/js/a12215ea.35e2776f.js
+++ b/assets/js/a12215ea.1ea3e4ef.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[98],{168:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(232)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.11/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.11/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/intro.md",version:"1.2.11",sidebar_label:"Intro",sidebar:"version-1.2.11/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[98],{168:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(228)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.11/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.11/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/intro.md",version:"1.2.11",sidebar_label:"Intro",sidebar:"version-1.2.11/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.11/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a3c17fac.4f9a964b.js b/assets/js/a3c17fac.4f9a964b.js
deleted file mode 100644
index 340f6e35..00000000
--- a/assets/js/a3c17fac.4f9a964b.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[99],{169:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"version-2.0.0/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-2.0.0/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_first_steps.md",version:"2.0.0"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a55d7fc8.3675ba14.js b/assets/js/a55d7fc8.3675ba14.js
deleted file mode 100644
index a9c9194b..00000000
--- a/assets/js/a55d7fc8.3675ba14.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[100],{170:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_upgrade",id:"version-1.2.8/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Shut things down",source:"@site/versioned_docs/version-1.2.8/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_upgrade.md",version:"1.2.8"},s=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Files",id:"checkupdate-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart Docker Containers",id:"restart-docker-containers",children:[]}],l={toc:s};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker_compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-files"},"Check/Update Files"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_value \n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this stop!  This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-docker-containers"},"Restart Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),p=d(n),m=r,b=p["".concat(i,".").concat(m)]||p[m]||u[m]||a;return n?o.a.createElement(b,c(c({ref:t},l),{},{components:n})):o.a.createElement(b,c({ref:t},l))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a55d7fc8.d2b40c2a.js b/assets/js/a55d7fc8.d2b40c2a.js
new file mode 100644
index 00000000..0f487ade
--- /dev/null
+++ b/assets/js/a55d7fc8.d2b40c2a.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[99],{169:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_upgrade",id:"version-1.2.8/components/docker_upgrade",isDocsHomePage:!1,title:"docker_upgrade",description:"Shut things down",source:"@site/versioned_docs/version-1.2.8/components/docker_upgrade.md",slug:"/components/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_upgrade.md",version:"1.2.8"},s=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Files",id:"checkupdate-files",children:[]},{value:"Select Release Version",id:"select-release-version",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart Docker Containers",id:"restart-docker-containers",children:[]}],l={toc:s};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker_compose down\n")),Object(a.b)("p",null,"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("h3",{id:"checkupdate-files"},"Check/Update Files"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"select-release-version"},"Select Release Version"),Object(a.b)("p",null,'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_value \n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"Do not forget this stop!  This applies for both development and release versions."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-docker-containers"},"Restart Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),d=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),p=d(n),m=r,b=p["".concat(i,".").concat(m)]||p[m]||u[m]||a;return n?o.a.createElement(b,c(c({ref:t},l),{},{components:n})):o.a.createElement(b,c({ref:t},l))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/a7459da2.03c88ff1.js b/assets/js/a7459da2.2d033572.js
similarity index 99%
rename from assets/js/a7459da2.03c88ff1.js
rename to assets/js/a7459da2.2d033572.js
index 659910c9..36e2558e 100644
--- a/assets/js/a7459da2.03c88ff1.js
+++ b/assets/js/a7459da2.2d033572.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[101],{171:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return b})),a.d(t,"toc",(function(){return c})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(232)),i={id:"pipeline_logstash",title:"Pipeline Logstash",sidebar_label:"Logstash"},b={unversionedId:"pipeline_logstash",id:"version-1.2.5/pipeline_logstash",isDocsHomePage:!1,title:"Pipeline Logstash",description:"Logstash",source:"@site/versioned_docs/version-1.2.5/pipeline_logstash.md",slug:"/pipeline_logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_logstash.md",version:"1.2.5",sidebar_label:"Logstash",sidebar:"version-1.2.5/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer"},next:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker"}},c=[{value:"Logstash Procedures",id:"logstash-procedures",children:[{value:"01-inputs.conf",id:"01-inputsconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"50-geoip-tagging.conf",id:"50-geoip-taggingconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"65-preferred-location-org.conf",id:"65-preferred-location-orgconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"99-outputs.conf -",id:"99-outputsconf--",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[{value:"ES fields",id:"es-fields",children:[]},{value:"Developer fields",id:"developer-fields",children:[]},{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly)",id:"source-fields-destination-fields-similarly",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly)",id:"source-science-registry-fields--destination-fields-similarly",children:[]},{value:"&quot;Preferred&quot; fields",id:"preferred-fields",children:[]},{value:"Values",id:"values",children:[]},{value:"Tstat Values",id:"tstat-values",children:[]}]}],o={toc:c};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(l.b)("wrapper",Object(n.a)({},o,a,{components:t,mdxType:"MDXLayout"}),Object(l.b)("h1",{id:"logstash"},"Logstash"),Object(l.b)("p",null,"These Logstash config files are in /etc/logstash/conf.d/"),Object(l.b)("h2",{id:"logstash-procedures"},"Logstash Procedures"),Object(l.b)("p",null,"The following steps are defined for logstash:"),Object(l.b)("h3",{id:"01-inputsconf"},"01-inputs.conf"),Object(l.b)("p",null,"reads flows from wherever the user wants to get data; in our case, from the netsage_deidentifer_raw rabbit queue."),Object(l.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(l.b)("p",null,"drops flows to or from private IP addresses;\nadds @injest_time;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions"),Object(l.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(l.b)("p",null,"adds a unique id based on the 5-tuple (src and dst ips and ports, and protocol) plus the sensor name. This ends up being called meta.id.\n(30-xxx.conf is not used yet)"),Object(l.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(l.b)("p",null,"stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(l.b)("p",null,"Notes: By default, 5-minute nfcapd files are assumed, and if less than 10.5 min have passed between the start of the current flow and the start of the last matching one, stitch the two together."),Object(l.b)("p",null,"Your logstash pipeline can have only 1 worker or aggregation is not going to work! "),Object(l.b)("h3",{id:"50-geoip-taggingconf"},"50-geoip-tagging.conf"),Object(l.b)("p",null,'if the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast";\nqueries the MaxMind GeoLite2-ASN database by IP to get src and dst Organizations (note that these are the organizations associated with the ASN);\nif the ASN in the flow header differs from that which MaxMind gives, the flow header\'s value is saved ',"[we need to not do this if it's a private ASN]",";\nqueries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes. "),Object(l.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(l.b)("p",null,"[this part is commented out right now: if a src or dst ASN matches a certain value, search only the corresponding lookup table.]","\nSearch lookup tables by IP to obtain member or customer organization names and overwrite the GeoIP ASN Organization where applicable.\nThis allows us to have entities which don't own their own ASs be listed as the src or dst Organization."),Object(l.b)("p",null,Object(l.b)("strong",{parentName:"p"},"Notes"),": Lookup tables are not stored in github."),Object(l.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(l.b)("p",null,"uses a fake geoip database containing Science Registry information to add, for src and dst, science discipline and role, science registry organization and location, etc;\nremoves scireg fields we don't actually need to save to elasticsearch."),Object(l.b)("p",null,Object(l.b)("strong",{parentName:"p"},"Notes"),": The science registry fake geoip database can be downloaded from scienceregistry.grnoc.iu.edu via a cron job. "),Object(l.b)("h3",{id:"65-preferred-location-orgconf"},"65-preferred-location-org.conf"),Object(l.b)("p",null,"copies science registry organization and/or location values, if they exist, to the preferred_organization and/or perferred_location fields."),Object(l.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(l.b)("p",null,"replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them. "),Object(l.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(l.b)("p",null,'removes information about Australian organizations (or, with modification, any organization that has privacy rules that require us not to identify them).\nIf the ASN is one listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(l.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(l.b)("p",null,"sets additional quick and easy fields. The first 2 use lookup files based on regexes.  Currently we have:\nsensor_group  = TACC, AMPATH, etc\nsensor_type    = Circuit, Archive, Exchange Point, or Regional Network\ncountry_scope = Domestic, International, or Mixed"),Object(l.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(l.b)("p",null,"does small misc. tasks at the end like rename, remove, or convert fields"),Object(l.b)("h3",{id:"99-outputsconf--"},"99-outputs.conf -"),Object(l.b)("p",null,"adds @exit_time and @processing_time,\nsends results to whereever the user wants them to go. In our case, it sends them to the netsage_archive_input queue (or netsage_ilight_input, etc) on netsage-elk1.grnoc.iu.edu. "),Object(l.b)("h3",{id:"final-stage"},"Final Stage"),Object(l.b)("p",null,"In our case, OmniSOC manages the last stage. Their logstash reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-",Object(l.b)("em",{parentName:"p"}," (or om-ns-ilight-"),", etc).  "),Object(l.b)("p",null,"This can be easily replicated with the following configuration though you'll need one for each feed/index."),Object(l.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(l.b)("pre",null,Object(l.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(l.b)("p",null,"Once the data is published in elastic, you can use the ",Object(l.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboard")," to visualize the data."),Object(l.b)("h2",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("h3",{id:"es-fields"},"ES fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06."),Object(l.b)("td",{parentName:"tr",align:null},"equivalent to an sql table")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"document id, set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))),Object(l.b)("h3",{id:"developer-fields"},"Developer fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"type"),Object(l.b)("td",{parentName:"tr",align:null},"flow"),Object(l.b)("td",{parentName:"tr",align:null},'Always "flow" for us. Other types may be "macy", etc.')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@injest_time"),Object(l.b)("td",{parentName:"tr",align:null},"2020-06-09T21:51:57.059Z"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline (99-outputs.conf)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @injest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.")))),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned flow id")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"Sflow, Netflow, or Tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor group")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor type")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly"},"Source Fields (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"ASN of the IP from geoip ASN database or the ASN from the flow header")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS of the IP from geoip ASN database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly"},"Source Science Registry Fields  (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford - ImageNet"),Object(l.b)("td",{parentName:"tr",align:null},"Resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"-"),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"CS. Intelligent Systems"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"."),Object(l.b)("td",{parentName:"tr",align:null},"Can be an array of projects ","[we may change this field name soon]")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's location, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"preferred-fields"},'"Preferred" fields'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Otherwise, they are the geoip values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"values"},"Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-values"},"Tstat Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))))}s.isMDXComponent=!0},232:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function b(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){l(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function c(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var o=r.a.createContext({}),s=function(e){var t=r.a.useContext(o),a=t;return e&&(a="function"==typeof e?e(t):b(b({},t),e)),a},d=function(e){var t=s(e.components);return r.a.createElement(o.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,o=c(e,["components","mdxType","originalType","parentName"]),d=s(a),m=n,u=d["".concat(i,".").concat(m)]||d[m]||p[m]||l;return a?r.a.createElement(u,b(b({ref:t},o),{},{components:a})):r.a.createElement(u,b({ref:t},o))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,i=new Array(l);i[0]=m;var b={};for(var c in t)hasOwnProperty.call(t,c)&&(b[c]=t[c]);b.originalType=e,b.mdxType="string"==typeof e?e:n,i[1]=b;for(var o=2;o<l;o++)i[o]=a[o];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[100],{170:function(e,t,a){"use strict";a.r(t),a.d(t,"frontMatter",(function(){return i})),a.d(t,"metadata",(function(){return b})),a.d(t,"toc",(function(){return c})),a.d(t,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(228)),i={id:"pipeline_logstash",title:"Pipeline Logstash",sidebar_label:"Logstash"},b={unversionedId:"pipeline_logstash",id:"version-1.2.5/pipeline_logstash",isDocsHomePage:!1,title:"Pipeline Logstash",description:"Logstash",source:"@site/versioned_docs/version-1.2.5/pipeline_logstash.md",slug:"/pipeline_logstash",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_logstash.md",version:"1.2.5",sidebar_label:"Logstash",sidebar:"version-1.2.5/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer"},next:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.5/devel/docker"}},c=[{value:"Logstash Procedures",id:"logstash-procedures",children:[{value:"01-inputs.conf",id:"01-inputsconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"50-geoip-tagging.conf",id:"50-geoip-taggingconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"65-preferred-location-org.conf",id:"65-preferred-location-orgconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"99-outputs.conf -",id:"99-outputsconf--",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[{value:"ES fields",id:"es-fields",children:[]},{value:"Developer fields",id:"developer-fields",children:[]},{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly)",id:"source-fields-destination-fields-similarly",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly)",id:"source-science-registry-fields--destination-fields-similarly",children:[]},{value:"&quot;Preferred&quot; fields",id:"preferred-fields",children:[]},{value:"Values",id:"values",children:[]},{value:"Tstat Values",id:"tstat-values",children:[]}]}],o={toc:c};function s(e){var t=e.components,a=Object(r.a)(e,["components"]);return Object(l.b)("wrapper",Object(n.a)({},o,a,{components:t,mdxType:"MDXLayout"}),Object(l.b)("h1",{id:"logstash"},"Logstash"),Object(l.b)("p",null,"These Logstash config files are in /etc/logstash/conf.d/"),Object(l.b)("h2",{id:"logstash-procedures"},"Logstash Procedures"),Object(l.b)("p",null,"The following steps are defined for logstash:"),Object(l.b)("h3",{id:"01-inputsconf"},"01-inputs.conf"),Object(l.b)("p",null,"reads flows from wherever the user wants to get data; in our case, from the netsage_deidentifer_raw rabbit queue."),Object(l.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(l.b)("p",null,"drops flows to or from private IP addresses;\nadds @injest_time;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions"),Object(l.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(l.b)("p",null,"adds a unique id based on the 5-tuple (src and dst ips and ports, and protocol) plus the sensor name. This ends up being called meta.id.\n(30-xxx.conf is not used yet)"),Object(l.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(l.b)("p",null,"stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(l.b)("p",null,"Notes: By default, 5-minute nfcapd files are assumed, and if less than 10.5 min have passed between the start of the current flow and the start of the last matching one, stitch the two together."),Object(l.b)("p",null,"Your logstash pipeline can have only 1 worker or aggregation is not going to work! "),Object(l.b)("h3",{id:"50-geoip-taggingconf"},"50-geoip-tagging.conf"),Object(l.b)("p",null,'if the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast";\nqueries the MaxMind GeoLite2-ASN database by IP to get src and dst Organizations (note that these are the organizations associated with the ASN);\nif the ASN in the flow header differs from that which MaxMind gives, the flow header\'s value is saved ',"[we need to not do this if it's a private ASN]",";\nqueries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes. "),Object(l.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(l.b)("p",null,"[this part is commented out right now: if a src or dst ASN matches a certain value, search only the corresponding lookup table.]","\nSearch lookup tables by IP to obtain member or customer organization names and overwrite the GeoIP ASN Organization where applicable.\nThis allows us to have entities which don't own their own ASs be listed as the src or dst Organization."),Object(l.b)("p",null,Object(l.b)("strong",{parentName:"p"},"Notes"),": Lookup tables are not stored in github."),Object(l.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(l.b)("p",null,"uses a fake geoip database containing Science Registry information to add, for src and dst, science discipline and role, science registry organization and location, etc;\nremoves scireg fields we don't actually need to save to elasticsearch."),Object(l.b)("p",null,Object(l.b)("strong",{parentName:"p"},"Notes"),": The science registry fake geoip database can be downloaded from scienceregistry.grnoc.iu.edu via a cron job. "),Object(l.b)("h3",{id:"65-preferred-location-orgconf"},"65-preferred-location-org.conf"),Object(l.b)("p",null,"copies science registry organization and/or location values, if they exist, to the preferred_organization and/or perferred_location fields."),Object(l.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(l.b)("p",null,"replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them. "),Object(l.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(l.b)("p",null,'removes information about Australian organizations (or, with modification, any organization that has privacy rules that require us not to identify them).\nIf the ASN is one listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(l.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(l.b)("p",null,"sets additional quick and easy fields. The first 2 use lookup files based on regexes.  Currently we have:\nsensor_group  = TACC, AMPATH, etc\nsensor_type    = Circuit, Archive, Exchange Point, or Regional Network\ncountry_scope = Domestic, International, or Mixed"),Object(l.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(l.b)("p",null,"does small misc. tasks at the end like rename, remove, or convert fields"),Object(l.b)("h3",{id:"99-outputsconf--"},"99-outputs.conf -"),Object(l.b)("p",null,"adds @exit_time and @processing_time,\nsends results to whereever the user wants them to go. In our case, it sends them to the netsage_archive_input queue (or netsage_ilight_input, etc) on netsage-elk1.grnoc.iu.edu. "),Object(l.b)("h3",{id:"final-stage"},"Final Stage"),Object(l.b)("p",null,"In our case, OmniSOC manages the last stage. Their logstash reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-",Object(l.b)("em",{parentName:"p"}," (or om-ns-ilight-"),", etc).  "),Object(l.b)("p",null,"This can be easily replicated with the following configuration though you'll need one for each feed/index."),Object(l.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(l.b)("pre",null,Object(l.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(l.b)("p",null,"Once the data is published in elastic, you can use the ",Object(l.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboard")," to visualize the data."),Object(l.b)("h2",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("h3",{id:"es-fields"},"ES fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06."),Object(l.b)("td",{parentName:"tr",align:null},"equivalent to an sql table")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"document id, set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))),Object(l.b)("h3",{id:"developer-fields"},"Developer fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"type"),Object(l.b)("td",{parentName:"tr",align:null},"flow"),Object(l.b)("td",{parentName:"tr",align:null},'Always "flow" for us. Other types may be "macy", etc.')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@injest_time"),Object(l.b)("td",{parentName:"tr",align:null},"2020-06-09T21:51:57.059Z"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline (99-outputs.conf)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @injest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.")))),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned flow id")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"Sflow, Netflow, or Tstat")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor group")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Assigned sensor type")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic, International, or Mixed, depending on countries of src and dst")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly"},"Source Fields (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"ASN of the IP from geoip ASN database or the ASN from the flow header")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of IP from geoip database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS of the IP from geoip ASN database")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly"},"Source Science Registry Fields  (Destination Fields similarly)"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford - ImageNet"),Object(l.b)("td",{parentName:"tr",align:null},"Resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"-"),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"CS. Intelligent Systems"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.projects"),Object(l.b)("td",{parentName:"tr",align:null},"."),Object(l.b)("td",{parentName:"tr",align:null},"Can be an array of projects ","[we may change this field name soon]")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's location, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"preferred-fields"},'"Preferred" fields'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, these are the SciReg values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Otherwise, they are the geoip values.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000"),Object(l.b)("td",{parentName:"tr",align:null})))),Object(l.b)("h3",{id:"values"},"Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in all the stitched flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-values"},"Tstat Values"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))))}s.isMDXComponent=!0},228:function(e,t,a){"use strict";a.d(t,"a",(function(){return d})),a.d(t,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function b(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){l(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function c(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var o=r.a.createContext({}),s=function(e){var t=r.a.useContext(o),a=t;return e&&(a="function"==typeof e?e(t):b(b({},t),e)),a},d=function(e){var t=s(e.components);return r.a.createElement(o.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},m=r.a.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,o=c(e,["components","mdxType","originalType","parentName"]),d=s(a),m=n,u=d["".concat(i,".").concat(m)]||d[m]||p[m]||l;return a?r.a.createElement(u,b(b({ref:t},o),{},{components:a})):r.a.createElement(u,b({ref:t},o))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,i=new Array(l);i[0]=m;var b={};for(var c in t)hasOwnProperty.call(t,c)&&(b[c]=t[c]);b.originalType=e,b.mdxType="string"==typeof e?e:n,i[1]=b;for(var o=2;o<l;o++)i[o]=a[o];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,a)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/aa98ec4a.ffbdd576.js b/assets/js/aa98ec4a.39b5137c.js
similarity index 98%
rename from assets/js/aa98ec4a.ffbdd576.js
rename to assets/js/aa98ec4a.39b5137c.js
index 822c39c3..891bf812 100644
--- a/assets/js/aa98ec4a.ffbdd576.js
+++ b/assets/js/aa98ec4a.39b5137c.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[102],{172:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),i=(n(0),n(232)),a={id:"docker",title:"Docker Guide",sidebar_label:"Docker Guide"},l={unversionedId:"devel/docker",id:"version-1.2.6/devel/docker",isDocsHomePage:!1,title:"Docker Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.6/devel/docker.md",slug:"/devel/docker",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/devel/docker.md",version:"1.2.6",sidebar_label:"Docker Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],c={toc:s};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(i.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(i.b)("p",null,"To select a released version please do the following."),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh tagValue")),Object(i.b)("p",null,"Example:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh v1.2.5")),Object(i.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(i.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(i.b)("h3",{id:"development-version"},"Development Version"),Object(i.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(i.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(i.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n")),Object(i.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(i.b)("h3",{id:"rabbit"},"Rabbit"),Object(i.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(i.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(i.b)("h3",{id:"importer"},"Importer"),Object(i.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(i.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(i.b)("h3",{id:"logstash"},"Logstash"),Object(i.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(i.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(i.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line:  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d")))),Object(i.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),b=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=b(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=b(n),d=o,h=p["".concat(a,".").concat(d)]||p[d]||u[d]||i;return n?r.a.createElement(h,l(l({ref:t},c),{},{components:n})):r.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[101],{171:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return b}));var o=n(3),r=n(7),i=(n(0),n(228)),a={id:"docker",title:"Docker Guide",sidebar_label:"Docker Guide"},l={unversionedId:"devel/docker",id:"version-1.2.6/devel/docker",isDocsHomePage:!1,title:"Docker Guide",description:"Docker Setup",source:"@site/versioned_docs/version-1.2.6/devel/docker.md",slug:"/devel/docker",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/devel/docker.md",version:"1.2.6",sidebar_label:"Docker Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Pipeline Logstash",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install"}},s=[{value:"Selecting a Version",id:"selecting-a-version",children:[{value:"Stable Release Version",id:"stable-release-version",children:[]},{value:"Development Version",id:"development-version",children:[]}]},{value:"Build Base Images",id:"build-base-images",children:[{value:"Build Using Source Code",id:"build-using-source-code",children:[]}]},{value:"Configuring the Containers",id:"configuring-the-containers",children:[{value:"Environment File",id:"environment-file",children:[]},{value:"Rabbit",id:"rabbit",children:[]},{value:"Importer",id:"importer",children:[]},{value:"Logstash",id:"logstash",children:[]},{value:"Optional: ElasticSearch and Kibana",id:"optional-elasticsearch-and-kibana",children:[]}]},{value:"Running the Containers",id:"running-the-containers",children:[{value:"Start the Containers",id:"start-the-containers",children:[]},{value:"Stop the Containers",id:"stop-the-containers",children:[]},{value:"Enter a Container Shell",id:"enter-a-container-shell",children:[]},{value:"View Container Logs",id:"view-container-logs",children:[]}]}],c={toc:s};function b(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h1",{id:"docker-setup"},"Docker Setup"),Object(i.b)("h2",{id:"selecting-a-version"},"Selecting a Version"),Object(i.b)("p",null,"We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  "),Object(i.b)("h3",{id:"stable-release-version"},"Stable Release Version"),Object(i.b)("p",null,"To select a released version please do the following."),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh tagValue")),Object(i.b)("p",null,"Example:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"scripts/docker_select_version.sh v1.2.5")),Object(i.b)("p",null,"If you run the script without any version specified it'll list all the current tags and prompt you to select a version."),Object(i.b)("p",null,"Once that's complete, all the instructions below are still applicable. "),Object(i.b)("h3",{id:"development-version"},"Development Version"),Object(i.b)("p",null,"If you wish to use the development version you are free to do so.  It is the default behavior on\nany git checkout.  Simply follow the directions below and setup your pipeline as instructed."),Object(i.b)("h2",{id:"build-base-images"},"Build Base Images"),Object(i.b)("p",null,"This is optional.  The image are published on docker hub, but if you'd like to incorporate local changes please follow the process below."),Object(i.b)("h3",{id:"build-using-source-code"},"Build Using Source Code"),Object(i.b)("p",null,"If you would like to build the ",Object(i.b)("em",{parentName:"p"},"importer")," container using the version of the pipeline scripts found in this GitHub repo then run the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose -f docker-compose.build.yml build\n")),Object(i.b)("h2",{id:"configuring-the-containers"},"Configuring the Containers"),Object(i.b)("h3",{id:"environment-file"},"Environment File"),Object(i.b)("p",null,"If you haven't done so already, copy env.example and update it to match your own settings:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env\n")),Object(i.b)("h3",{id:"rabbit"},"Rabbit"),Object(i.b)("p",null,"This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set\nhere should be consistent with the config for the logstash and importer "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"RABBITMQ_ERLANG_COOKIE='secret cookie'\nRABBIT_HOST=rabbit\nRABBITMQ_DEFAULT_USER=guest\nRABBITMQ_DEFAULT_PASS=guest\ndiscovery.type=single-node\n")),Object(i.b)("p",null,"Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it's set to rabbit"),Object(i.b)("h3",{id:"importer"},"Importer"),Object(i.b)("p",null,"The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ ",Object(i.b)("strong",{parentName:"p"},"NOTE: Changes will require you to rebuild the container")),Object(i.b)("h3",{id:"logstash"},"Logstash"),Object(i.b)("p",null,"Define the input rabbit queue.  This should match the importer output queue"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_input_host=rabbit\nrabbitmq_input_username=guest\nrabbitmq_input_pw=guest\n\n")),Object(i.b)("p",null,"Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(i.b)("h3",{id:"optional-elasticsearch-and-kibana"},"Optional: ElasticSearch and Kibana"),Object(i.b)("p",null,"You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Uncomment the following lines in conf-logstash/99-outputs.conf:")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'elasticsearch {\n    hosts => ["elasticsearch"]\n    index => "netsage_flow-%{+YYYY.MM.dd}"\n}\n')),Object(i.b)("ol",{start:2},Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Comment out the ",Object(i.b)("inlineCode",{parentName:"p"},"rabbitmq {...}")," block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.")),Object(i.b)("li",{parentName:"ol"},Object(i.b)("p",{parentName:"li"},"Run the containers using the following line:  ",Object(i.b)("inlineCode",{parentName:"p"},"docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d")))),Object(i.b)("h2",{id:"running-the-containers"},"Running the Containers"),Object(i.b)("h3",{id:"start-the-containers"},"Start the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d \n")),Object(i.b)("h3",{id:"stop-the-containers"},"Stop the Containers"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose stop && docker-compose rm \n")),Object(i.b)("h3",{id:"enter-a-container-shell"},"Enter a Container Shell"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose exec logstash bash     #bash shell in logstash container\ndocker-compose exec importer bash     #bash shell in importer container\ndocker-compose exec rabbit bash       #bash shell in rabbit container\n")),Object(i.b)("h3",{id:"view-container-logs"},"View Container Logs"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs -f              #view logs for all containers \ndocker-compose logs -f logstash     #view logs for logstash container\ndocker-compose logs -f importer     #view logs for importer container\ndocker-compose logs -f rabbit       #view logs for rabbit container\n")))}b.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),r=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),b=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=b(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=b(n),d=o,h=p["".concat(a,".").concat(d)]||p[d]||u[d]||i;return n?r.a.createElement(h,l(l({ref:t},c),{},{components:n})):r.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/aafd28b2.21d293f7.js b/assets/js/aafd28b2.fb1f547f.js
similarity index 99%
rename from assets/js/aafd28b2.21d293f7.js
rename to assets/js/aafd28b2.fb1f547f.js
index 13c016a0..67ccef3e 100644
--- a/assets/js/aafd28b2.21d293f7.js
+++ b/assets/js/aafd28b2.fb1f547f.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[103],{173:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.7/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",version:"1.2.7",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash)",id:"installing-the-pipeline-importer-and-logstash",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash"},"Installing the Pipeline (Importer and Logstash)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers is set to 1 (unless you have removed the aggregation logstash conf)."),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like --\x3e\n         <sensor>Netflow Sensor 1d</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>netflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash daily. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=o,h=u["".concat(l,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=b;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[102],{172:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),i=(n(0),n(228)),l={id:"bare_metal_install",title:"NetSage Flow Processing Pipeline Installation Guide",sidebar_label:"Server Installation Guide"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.7/deploy/bare_metal_install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",version:"1.2.7",sidebar_label:"Server Installation Guide",sidebar:"version-1.2.7/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.7/deploy/choose_install"},next:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash)",id:"installing-the-pipeline-importer-and-logstash",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash"},"Installing the Pipeline (Importer and Logstash)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers is set to 1 (unless you have removed the aggregation logstash conf)."),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like --\x3e\n         <sensor>Netflow Sensor 1d</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>netflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash daily. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=o,h=u["".concat(l,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(h,r(r({ref:t},c),{},{components:n})):a.a.createElement(h,r({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,l=new Array(i);l[0]=b;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:o,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/abbf3a21.97ae394b.js b/assets/js/abbf3a21.2995f8fb.js
similarity index 97%
rename from assets/js/abbf3a21.97ae394b.js
rename to assets/js/abbf3a21.2995f8fb.js
index 06d716eb..b795b6d0 100644
--- a/assets/js/abbf3a21.97ae394b.js
+++ b/assets/js/abbf3a21.2995f8fb.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[104],{174:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.8/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.8/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_env.md",version:"1.2.8"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[103],{173:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.8/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.8/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.8/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_env.md",version:"1.2.8"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ac3b7d69.fd3bbc4c.js b/assets/js/ac3b7d69.6126d7a7.js
similarity index 98%
rename from assets/js/ac3b7d69.fd3bbc4c.js
rename to assets/js/ac3b7d69.6126d7a7.js
index 57716aed..c5915fa6 100644
--- a/assets/js/ac3b7d69.fd3bbc4c.js
+++ b/assets/js/ac3b7d69.6126d7a7.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[105],{175:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return d}));var a=o(3),n=o(7),r=(o(0),o(232)),i={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.7/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",version:"1.2.7",sidebar_label:"Docker Advanced",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. docker-compose.override.yml",id:"1-docker-composeoverrideyml",children:[]},{value:"2.  netsage_override.xml",id:"2--netsage_overridexml",children:[]},{value:"3. Environment file",id:"3-environment-file",children:[]},{value:"Select which version to run",id:"select-which-version-to-run",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"For Tstat Data",id:"for-tstat-data",children:[]},{value:"To Customize Logstash Java Settings",id:"to-customize-logstash-java-settings",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"To Export Data to an NFS Volume",id:"to-export-data-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,o=Object(n.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,"Please first read the Docker Simple installation guide in detail. This guide will build on top of that."),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out while waiting for flows from sensors B-D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-docker-composeoverrideyml"},"1. docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--netsage_overridexml"},"2.  netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-environment-file"},"3. Environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},'exampleSensorName="Example New York sFlow"\n')),Object(r.b)("h3",{id:"select-which-version-to-run"},"Select which version to run"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable (or have code not ready for use) or a 'Released' version. If you're opting to use the Develop version, then simply skip the version selection step."),Object(r.b)("div",{className:"admonition admonition-caution alert alert--warning"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"16",height:"16",viewBox:"0 0 16 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"}))),"caution")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Warning! it is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update."))),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},'  git fetch \n  git checkout "tag name"\n  ./scripts/docker_select_version.sh\n')),Object(r.b)("p",null,'Replace "tag name" with the version you intend to use, eg, "v1.2.5".  When prompted by the script, select the same version.'),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After selecting the docker version to run, you can start the new collector by running the following line, using the collector name:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"for-tstat-data"},"For Tstat Data"),Object(r.b)("p",null,'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",null,"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),"."),Object(r.b)("h2",{id:"to-customize-logstash-java-settings"},"To Customize Logstash Java Settings"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"to-export-data-to-an-nfs-volume"},"To Export Data to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure to update all the paths correctly for the next release manually."))))}d.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return h}));var a=o(0),n=o.n(a);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){r(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,a,n=function(e,t){if(null==e)return{};var o,a,n={},r=Object.keys(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var c=n.a.createContext({}),d=function(e){var t=n.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=d(e.components);return n.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.a.createElement(n.a.Fragment,{},t)}},m=n.a.forwardRef((function(e,t){var o=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(o),m=a,h=p["".concat(i,".").concat(m)]||p[m]||u[m]||r;return o?n.a.createElement(h,l(l({ref:t},c),{},{components:o})):n.a.createElement(h,l({ref:t},c))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=o.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=o[c];return n.a.createElement.apply(null,i)}return n.a.createElement.apply(null,o)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[104],{174:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return d}));var a=o(3),n=o(7),r=(o(0),o(228)),i={id:"docker_install_advanced",title:"Docker Advanced Installation Guide",sidebar_label:"Docker Advanced"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.7/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Installation Guide",description:"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.",source:"@site/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",version:"1.2.7",sidebar_label:"Docker Advanced",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Default Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"1. docker-compose.override.yml",id:"1-docker-composeoverrideyml",children:[]},{value:"2.  netsage_override.xml",id:"2--netsage_overridexml",children:[]},{value:"3. Environment file",id:"3-environment-file",children:[]},{value:"Select which version to run",id:"select-which-version-to-run",children:[]},{value:"Running the new collector",id:"running-the-new-collector",children:[]}]},{value:"For Tstat Data",id:"for-tstat-data",children:[]},{value:"To Customize Logstash Java Settings",id:"to-customize-logstash-java-settings",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"To Export Data to an NFS Volume",id:"to-export-data-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,o=Object(n.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."),Object(r.b)("p",null,"Please first read the Docker Simple installation guide in detail. This guide will build on top of that."),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out while waiting for flows from sensors B-D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"1-docker-composeoverrideyml"},"1. docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"2--netsage_overridexml"},"2.  netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"3-environment-file"},"3. Environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},'exampleSensorName="Example New York sFlow"\n')),Object(r.b)("h3",{id:"select-which-version-to-run"},"Select which version to run"),Object(r.b)("p",null,"Before continuing you need to choose if you are going to be use the 'Develop' version which has the latest changes but might be a bit less stable (or have code not ready for use) or a 'Released' version. If you're opting to use the Develop version, then simply skip the version selection step."),Object(r.b)("div",{className:"admonition admonition-caution alert alert--warning"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"16",height:"16",viewBox:"0 0 16 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"}))),"caution")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Warning! it is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update."))),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},'  git fetch \n  git checkout "tag name"\n  ./scripts/docker_select_version.sh\n')),Object(r.b)("p",null,'Replace "tag name" with the version you intend to use, eg, "v1.2.5".  When prompted by the script, select the same version.'),Object(r.b)("h3",{id:"running-the-new-collector"},"Running the new collector"),Object(r.b)("p",null,"After selecting the docker version to run, you can start the new collector by running the following line, using the collector name:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"for-tstat-data"},"For Tstat Data"),Object(r.b)("p",null,'Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline\'s ingest rabbit queue (named "netsage_deidentifier_raw").  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.'),Object(r.b)("p",null,"Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport.git"},"https://github.com/netsage-project/tstat-transport.git"),"."),Object(r.b)("h2",{id:"to-customize-logstash-java-settings"},"To Customize Logstash Java Settings"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"to-export-data-to-an-nfs-volume"},"To Export Data to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure to update all the paths correctly for the next release manually."))))}d.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return h}));var a=o(0),n=o.n(a);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){r(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,a,n=function(e,t){if(null==e)return{};var o,a,n={},r=Object.keys(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)o=r[a],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var c=n.a.createContext({}),d=function(e){var t=n.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=d(e.components);return n.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.a.createElement(n.a.Fragment,{},t)}},m=n.a.forwardRef((function(e,t){var o=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(o),m=a,h=p["".concat(i,".").concat(m)]||p[m]||u[m]||r;return o?n.a.createElement(h,l(l({ref:t},c),{},{components:o})):n.a.createElement(h,l({ref:t},c))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=o.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=o[c];return n.a.createElement.apply(null,i)}return n.a.createElement.apply(null,o)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ad308aff.9d4faea0.js b/assets/js/ad308aff.8a927dcc.js
similarity index 96%
rename from assets/js/ad308aff.9d4faea0.js
rename to assets/js/ad308aff.8a927dcc.js
index fe5647d3..725f329c 100644
--- a/assets/js/ad308aff.9d4faea0.js
+++ b/assets/js/ad308aff.8a927dcc.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[106],{176:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.8/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.8/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/importer.md",version:"1.2.8",sidebar_label:"Importer",sidebar:"version-1.2.8/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[105],{175:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-1.2.8/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-1.2.8/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/importer.md",version:"1.2.8",sidebar_label:"Importer",sidebar:"version-1.2.8/Pipeline",previous:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("p",null,"NOTE: The importer will be deprecated in the future and replace with a logstash operation."),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},f=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),f=r,m=u["".concat(a,".").concat(f)]||u[f]||d[f]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=f;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ae8a1364.d1aa581b.js b/assets/js/ae8a1364.4da8b56a.js
similarity index 96%
rename from assets/js/ae8a1364.d1aa581b.js
rename to assets/js/ae8a1364.4da8b56a.js
index 6607a473..c1343111 100644
--- a/assets/js/ae8a1364.d1aa581b.js
+++ b/assets/js/ae8a1364.4da8b56a.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[107],{177:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.10/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.10/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_first_steps.md",version:"1.2.10"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[106],{176:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.10/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.10/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_first_steps.md",version:"1.2.10"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/b1bfe9f5.13251023.js b/assets/js/b1bfe9f5.592f8cc1.js
similarity index 96%
rename from assets/js/b1bfe9f5.13251023.js
rename to assets/js/b1bfe9f5.592f8cc1.js
index d0602e2c..88b5c0ef 100644
--- a/assets/js/b1bfe9f5.13251023.js
+++ b/assets/js/b1bfe9f5.592f8cc1.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[109],{179:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.8/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.8/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/nfdump.md",version:"1.2.8",sidebar_label:"Nfdump",sidebar:"version-1.2.8/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[107],{177:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Nfdump",sidebar_label:"Nfdump"},s={unversionedId:"pipeline/nfdump",id:"version-1.2.8/pipeline/nfdump",isDocsHomePage:!1,title:"Nfdump",description:"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.8/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/nfdump.md",version:"1.2.8",sidebar_label:"Nfdump",sidebar:"version-1.2.8/Pipeline",previous:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/importer"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),l=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},f=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=l(n),f=r,m=d["".concat(i,".").concat(f)]||d[f]||u[f]||a;return n?o.a.createElement(m,s(s({ref:t},c),{},{components:n})):o.a.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/b3c114f4.e41fb427.js b/assets/js/b3c114f4.28ba7bd1.js
similarity index 96%
rename from assets/js/b3c114f4.e41fb427.js
rename to assets/js/b3c114f4.28ba7bd1.js
index 779964b5..1ed8ea03 100644
--- a/assets/js/b3c114f4.e41fb427.js
+++ b/assets/js/b3c114f4.28ba7bd1.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[110],{180:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.9/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.9/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_first_steps.md",version:"1.2.9"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[108],{178:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.9/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.9/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_first_steps.md",version:"1.2.9"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/b56f03c6.3836580d.js b/assets/js/b56f03c6.3836580d.js
deleted file mode 100644
index 1e3fb318..00000000
--- a/assets/js/b56f03c6.3836580d.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[111],{181:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"version-2.0.0/deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-2.0.0/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/bare_metal_install.md",version:"2.0.0",sidebar_label:"Manual Installation",sidebar:"version-2.0.0/Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/b83a1049.1cf10416.js b/assets/js/b83a1049.f6464bc0.js
similarity index 96%
rename from assets/js/b83a1049.1cf10416.js
rename to assets/js/b83a1049.f6464bc0.js
index 699567cf..78ca939c 100644
--- a/assets/js/b83a1049.1cf10416.js
+++ b/assets/js/b83a1049.f6464bc0.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[112],{182:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docker_dev_tag",title:"How to Tag a New Release",sidebar_label:"Taggin a Release"},l={unversionedId:"devel/docker_dev_tag",id:"version-1.2.10/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Tag a New Release",description:"To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images.",source:"@site/versioned_docs/version-1.2.10/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/tag.md",version:"1.2.10",sidebar_label:"Taggin a Release",sidebar:"version-1.2.10/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus"}},s=[{value:"In Github, Create a Release/Tag",id:"in-github-create-a-releasetag",children:[]},{value:"To Build and Push an Importer Image Manually",id:"to-build-and-push-an-importer-image-manually",children:[]},{value:"With Automation",id:"with-automation",children:[]},{value:"Versioned Docs",id:"versioned-docs",children:[]},{value:"New Version of Nfdump",id:"new-version-of-nfdump",children:[]},{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images."),Object(a.b)("h2",{id:"in-github-create-a-releasetag"},"In Github, Create a Release/Tag"),Object(a.b)("p",null,"Be sure to copy info from the Changes file into the Release description."),Object(a.b)("p",null,"Do this first ???"),Object(a.b)("h2",{id:"to-build-and-push-an-importer-image-manually"},"To Build and Push an Importer Image Manually"),Object(a.b)("p",null,"Git clone the pipeline project and have the ?? branch checked out. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ docker-compose build\n$ docker login\n$ docker push $image:$tag\n")),Object(a.b)("p",null,"This will build the image and push it to Docker Hub."),Object(a.b)("p",null,"The person doing this has to have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("h2",{id:"with-automation"},"With Automation"),Object(a.b)("h2",{id:"versioned-docs"},"Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged. See the Docusaurus guide. "),Object(a.b)("p",null,"Does this have to happen before Building the image ??"),Object(a.b)("h2",{id:"new-version-of-nfdump"},"New Version of Nfdump"),Object(a.b)("p",null,"If a new version of dfdump has been released that we need,\n????"),Object(a.b)("h2",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),b=r,f=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[109],{179:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_dev_tag",title:"How to Tag a New Release",sidebar_label:"Taggin a Release"},l={unversionedId:"devel/docker_dev_tag",id:"version-1.2.10/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Tag a New Release",description:"To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images.",source:"@site/versioned_docs/version-1.2.10/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/tag.md",version:"1.2.10",sidebar_label:"Taggin a Release",sidebar:"version-1.2.10/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/1.2.10/devel/docusaurus"}},s=[{value:"In Github, Create a Release/Tag",id:"in-github-create-a-releasetag",children:[]},{value:"To Build and Push an Importer Image Manually",id:"to-build-and-push-an-importer-image-manually",children:[]},{value:"With Automation",id:"with-automation",children:[]},{value:"Versioned Docs",id:"versioned-docs",children:[]},{value:"New Version of Nfdump",id:"new-version-of-nfdump",children:[]},{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images."),Object(a.b)("h2",{id:"in-github-create-a-releasetag"},"In Github, Create a Release/Tag"),Object(a.b)("p",null,"Be sure to copy info from the Changes file into the Release description."),Object(a.b)("p",null,"Do this first ???"),Object(a.b)("h2",{id:"to-build-and-push-an-importer-image-manually"},"To Build and Push an Importer Image Manually"),Object(a.b)("p",null,"Git clone the pipeline project and have the ?? branch checked out. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ docker-compose build\n$ docker login\n$ docker push $image:$tag\n")),Object(a.b)("p",null,"This will build the image and push it to Docker Hub."),Object(a.b)("p",null,"The person doing this has to have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("h2",{id:"with-automation"},"With Automation"),Object(a.b)("h2",{id:"versioned-docs"},"Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged. See the Docusaurus guide. "),Object(a.b)("p",null,"Does this have to happen before Building the image ??"),Object(a.b)("h2",{id:"new-version-of-nfdump"},"New Version of Nfdump"),Object(a.b)("p",null,"If a new version of dfdump has been released that we need,\n????"),Object(a.b)("h2",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),b=r,f=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ba55e963.5bed7840.js b/assets/js/ba55e963.db919392.js
similarity index 95%
rename from assets/js/ba55e963.5bed7840.js
rename to assets/js/ba55e963.db919392.js
index 1bb1256e..876747ef 100644
--- a/assets/js/ba55e963.5bed7840.js
+++ b/assets/js/ba55e963.db919392.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[113],{183:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return p})),t.d(n,"toc",(function(){return i})),t.d(n,"default",(function(){return s}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},p={unversionedId:"components/docker_pipeline",id:"components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/docs/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/next/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_pipeline.md",version:"current"},i=[],l={toc:i};function s(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}s.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function p(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var l=o.a.createContext({}),s=function(e){var n=o.a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):p(p({},n),e)),t},u=function(e){var n=s(e.components);return o.a.createElement(l.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,l=i(e,["components","mdxType","originalType","parentName"]),u=s(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,p(p({ref:n},l),{},{components:t})):o.a.createElement(m,p({ref:n},l))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var p={};for(var i in n)hasOwnProperty.call(n,i)&&(p[i]=n[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,c[1]=p;for(var l=2;l<a;l++)c[l]=t[l];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[110],{180:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return p})),t.d(n,"toc",(function(){return i})),t.d(n,"default",(function(){return s}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},p={unversionedId:"components/docker_pipeline",id:"components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/docs/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/next/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_pipeline.md",version:"current"},i=[],l={toc:i};function s(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(a.b)("p",null,"Run all commands from the netsage-pipeline/ directory."))}s.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function p(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var l=o.a.createContext({}),s=function(e){var n=o.a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):p(p({},n),e)),t},u=function(e){var n=s(e.components);return o.a.createElement(l.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,l=i(e,["components","mdxType","originalType","parentName"]),u=s(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,p(p({ref:n},l),{},{components:t})):o.a.createElement(m,p({ref:n},l))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var p={};for(var i in n)hasOwnProperty.call(n,i)&&(p[i]=n[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,c[1]=p;for(var l=2;l<a;l++)c[l]=t[l];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/bb237b1e.9e57bcc8.js b/assets/js/bb237b1e.ea30ebd1.js
similarity index 98%
rename from assets/js/bb237b1e.9e57bcc8.js
rename to assets/js/bb237b1e.ea30ebd1.js
index 0becdbcc..fb137856 100644
--- a/assets/js/bb237b1e.9e57bcc8.js
+++ b/assets/js/bb237b1e.ea30ebd1.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[114],{184:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.7/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.7/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/logstash.md",version:"1.2.7",sidebar_label:"Logstash",sidebar:"version-1.2.7/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf are not saved in github. You will need to provide these yourself or ask us for any we may have compliled. ")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("p",null,"Also adds a unique id (es_doc_id) based on meta.id plus the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates."),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In GlobalNOC/Netsage Project's case, the output filter writes the flows to a RabbitMQ queue on another host and the last stage is a separate logstash pipeline on that host. The latter reads flows from the queue and sends it into elasticsearch. "),Object(o.b)("p",null,"This can be easily replicated with the following configuration, for example, though you'll need one for each Rabbit queue/sensor/index. A mapping template can be specified here."),Object(o.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(o.b)("p",null,'Note: This should be updated to use a mapping template, for setting field types, and possibly using es_doc_id as the elasticsearch document id. As of Nov 2020, we are moving toward saving all fields as type "keyword", "float", or "long" only. '),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},f=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=d(n),f=i,b=p["".concat(s,".").concat(f)]||p[f]||u[f]||o;return n?a.a.createElement(b,r(r({ref:t},c),{},{components:n})):a.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=f;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<o;c++)s[c]=n[c];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[111],{181:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-1.2.7/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.)',source:"@site/versioned_docs/version-1.2.7/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/logstash.md",version:"1.2.7",sidebar_label:"Logstash",sidebar:"version-1.2.7/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/elastic"}},l=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],c={toc:l};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one "logstash pipeline" at a time, but that will not be discussed here.) '),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters". These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf are not saved in github. You will need to provide these yourself or ask us for any we may have compliled. ")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\ndoes some data type conversions;\nadds @ingest_time (this is mainly for developers)."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("p",null,"Also adds a unique id (es_doc_id) based on meta.id plus the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates."),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product includes GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In GlobalNOC/Netsage Project's case, the output filter writes the flows to a RabbitMQ queue on another host and the last stage is a separate logstash pipeline on that host. The latter reads flows from the queue and sends it into elasticsearch. "),Object(o.b)("p",null,"This can be easily replicated with the following configuration, for example, though you'll need one for each Rabbit queue/sensor/index. A mapping template can be specified here."),Object(o.b)("p",null,"Naturally the hosts for rabbit and elastic will need to be updated accordingly."),Object(o.b)("p",null,'Note: This should be updated to use a mapping template, for setting field types, and possibly using es_doc_id as the elasticsearch document id. As of Nov 2020, we are moving toward saving all fields as type "keyword", "float", or "long" only. '),Object(o.b)("pre",null,Object(o.b)("code",{parentName:"pre"},'input {\n  rabbitmq {\n    host => \'localhost\'\n    user => \'guest\'\n    password => "${rabbitmq_pass}"\n    exchange => \'netsage.direct\'\n    key =>   XXXXXXX\'\n    queue => \'netsage\'\n    durable => true\n    subscription_retry_interval_seconds => 5\n    connection_timeout => 10000\n  }\n}\nfilter {\n  if [@metadata][rabbitmq_properties][timestamp] {\n    date {\n      match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]\n    }\n  }\n}\n\noutput {\n    elasticsearch {\n      hosts => [\n          "https://CHANGEME1",\n          "https://CHANGEME2"\n      ]\n      user => "logstash"\n      password => "${logstash_elasticsearch_password}"\n      cacert => "/etc/logstash/ca.crt"\n      index => "om-ns-netsage"\n      template_overwrite => true\n      failure_type_logging_whitelist => []\n      action => index\n      #ssl_certificate_verification => false\n    }\n}\n')),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return b}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},f=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=d(n),f=i,b=p["".concat(s,".").concat(f)]||p[f]||u[f]||o;return n?a.a.createElement(b,r(r({ref:t},c),{},{components:n})):a.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=f;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var c=2;c<o;c++)s[c]=n[c];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}f.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/bd6c6f88.e87a9bd6.js b/assets/js/bd6c6f88.8d988769.js
similarity index 98%
rename from assets/js/bd6c6f88.e87a9bd6.js
rename to assets/js/bd6c6f88.8d988769.js
index caa23661..84555094 100644
--- a/assets/js/bd6c6f88.e87a9bd6.js
+++ b/assets/js/bd6c6f88.8d988769.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[115],{185:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(232)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/docs/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/next/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/logstash.md",version:"current",sidebar_label:"Logstash",sidebar:"Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/next/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[112],{182:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/docs/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/next/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/logstash.md",version:"current",sidebar_label:"Logstash",sidebar:"Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/next/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/bed7ab4e.6b5e2485.js b/assets/js/bed7ab4e.fea77ca6.js
similarity index 99%
rename from assets/js/bed7ab4e.6b5e2485.js
rename to assets/js/bed7ab4e.fea77ca6.js
index 73e05682..c1e39cb3 100644
--- a/assets/js/bed7ab4e.6b5e2485.js
+++ b/assets/js/bed7ab4e.fea77ca6.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[116],{186:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(232)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.8/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.8/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/elastic_search.md",version:"1.2.8",sidebar_label:"Elasticsearch",sidebar:"version-1.2.8/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},232:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[113],{183:function(t,e,a){"use strict";a.r(e),a.d(e,"frontMatter",(function(){return b})),a.d(e,"metadata",(function(){return i})),a.d(e,"toc",(function(){return c})),a.d(e,"default",(function(){return s}));var n=a(3),r=a(7),l=(a(0),a(228)),b={id:"elastic",title:"Elasticsearch",sidebar_label:"Elasticsearch"},i={unversionedId:"pipeline/elastic",id:"version-1.2.8/pipeline/elastic",isDocsHomePage:!1,title:"Elasticsearch",description:"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.",source:"@site/versioned_docs/version-1.2.8/pipeline/elastic_search.md",slug:"/pipeline/elastic",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/elastic_search.md",version:"1.2.8",sidebar_label:"Elasticsearch",sidebar:"version-1.2.8/Pipeline",previous:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/logstash"},next:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install"}},c=[{value:"Flow fields",id:"flow-fields",children:[]},{value:"Source Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-fields-destination-fields-similarly-with-dst",children:[]},{value:"Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)",id:"source-science-registry-fields--destination-fields-similarly-with-dst",children:[]},{value:"Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)",id:"source-preferred-fields-destination-fields-similarly-with-dst",children:[]},{value:"Value Fields",id:"value-fields",children:[]},{value:"Tstat Value Fields",id:"tstat-value-fields",children:[]},{value:"Developer Fields",id:"developer-fields",children:[]},{value:"Elasticsearch Fields",id:"elasticsearch-fields",children:[]}],d={toc:c};function s(t){var e=t.components,a=Object(r.a)(t,["components"]);return Object(l.b)("wrapper",Object(n.a)({},d,a,{components:e,mdxType:"MDXLayout"}),Object(l.b)("p",null,"Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."),Object(l.b)("h3",{id:"flow-fields"},"Flow fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"start"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:53.808"),Object(l.b)("td",{parentName:"tr",align:null},"Start time of the flow (first packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"end"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 17:39:57.699"),Object(l.b)("td",{parentName:"tr",align:null},"End time of the flow   (last packet seen)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.id"),Object(l.b)("td",{parentName:"tr",align:null},"a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779"),Object(l.b)("td",{parentName:"tr",align:null},"Id of the flow (hash of 5-tuple + Sensor name)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.flow_type"),Object(l.b)("td",{parentName:"tr",align:null},"sflow"),Object(l.b)("td",{parentName:"tr",align:null},"'sflow', 'netflow', or 'tstat'")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.protocol"),Object(l.b)("td",{parentName:"tr",align:null},"tcp"),Object(l.b)("td",{parentName:"tr",align:null},"Protocol used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_id"),Object(l.b)("td",{parentName:"tr",align:null},"snvl2-pw-sw-1-mgmt-2.cenic.net"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor name (set in importer config, may not always be a hostname)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_group"),Object(l.b)("td",{parentName:"tr",align:null},"CENIC"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor group, usually the network")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.sensor_type"),Object(l.b)("td",{parentName:"tr",align:null},"Regional Network"),Object(l.b)("td",{parentName:"tr",align:null},"Sensor type ('Circuit', 'Regional Network', etc)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.country_scope"),Object(l.b)("td",{parentName:"tr",align:null},"Domestic"),Object(l.b)("td",{parentName:"tr",align:null},"'Domestic', 'International', or 'Mixed', depending on countries of src and dst")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.is_network_testing"),Object(l.b)("td",{parentName:"tr",align:null},"no"),Object(l.b)("td",{parentName:"tr",align:null},"'yes' if discipline is 'CS.Network Testing and Monitoring' or port is one used for PerfSonar: 5001, 5101, or 5201")))),Object(l.b)("h3",{id:"source-fields-destination-fields-similarly-with-dst"},'Source Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ip"),Object(l.b)("td",{parentName:"tr",align:null},"171.64.68.x"),Object(l.b)("td",{parentName:"tr",align:null},"deidentified IP address")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_port"),Object(l.b)("td",{parentName:"tr",align:null},"80"),Object(l.b)("td",{parentName:"tr",align:null},"port used")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_asn"),Object(l.b)("td",{parentName:"tr",align:null},"32"),Object(l.b)("td",{parentName:"tr",align:null},"Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_organization"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"organization that owns the AS from the CAIDA ASN-Organization database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.423"),Object(l.b)("td",{parentName:"tr",align:null},"latitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.164"),Object(l.b)("td",{parentName:"tr",align:null},"longitude of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_country_name"),Object(l.b)("td",{parentName:"tr",align:null},"United States"),Object(l.b)("td",{parentName:"tr",align:null},"country of the IP from the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_continent"),Object(l.b)("td",{parentName:"tr",align:null},"North America"),Object(l.b)("td",{parentName:"tr",align:null},"continent of the IP the MaxMind GeoIP City database")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_ifindex"),Object(l.b)("td",{parentName:"tr",align:null},"166"),Object(l.b)("td",{parentName:"tr",align:null},"the index of the interface the flow came into")))),Object(l.b)("h3",{id:"source-science-registry-fields--destination-fields-similarly-with-dst"},'Source Science Registry Fields  (Destination Fields similarly with "dst")'),Object(l.b)("p",null,"The ",Object(l.b)("a",{parentName:"p",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.discipline"),Object(l.b)("td",{parentName:"tr",align:null},"MPS.Physics.High Energy"),Object(l.b)("td",{parentName:"tr",align:null},"The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.role"),Object(l.b)("td",{parentName:"tr",align:null},"Storage"),Object(l.b)("td",{parentName:"tr",align:null},"Role that the host plays")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_name"),Object(l.b)("td",{parentName:"tr",align:null},"Boston University (BU)"),Object(l.b)("td",{parentName:"tr",align:null},"The organization the manages and/or uses the resource, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.org_abbr"),Object(l.b)("td",{parentName:"tr",align:null},"Boston U"),Object(l.b)("td",{parentName:"tr",align:null},"A shorter name for the organization. May not be the official abbreviation.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource"),Object(l.b)("td",{parentName:"tr",align:null},"BU - ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},"Descriptive resource name from SciReg")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.resource_abbr"),Object(l.b)("td",{parentName:"tr",align:null}),Object(l.b)("td",{parentName:"tr",align:null},"Resource abbreviation (if any)")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.project_names"),Object(l.b)("td",{parentName:"tr",align:null},"ATLAS"),Object(l.b)("td",{parentName:"tr",align:null},'"Projects" that the resource is part of')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.latitude"),Object(l.b)("td",{parentName:"tr",align:null},"37.4178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's latitude, as listed in the Science Registry")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.scireg.src.longitude"),Object(l.b)("td",{parentName:"tr",align:null},"-122.178"),Object(l.b)("td",{parentName:"tr",align:null},"Resource's longitude, as listed in the Science Registry")))),Object(l.b)("h3",{id:"source-preferred-fields-destination-fields-similarly-with-dst"},'Source "Preferred" Fields (Destination Fields similarly with "dst")'),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_org"),Object(l.b)("td",{parentName:"tr",align:null},"Stanford University"),Object(l.b)("td",{parentName:"tr",align:null},"If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lat"),Object(l.b)("td",{parentName:"tr",align:null},"37.417800"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"meta.src_preferred_location.lon"),Object(l.b)("td",{parentName:"tr",align:null},"-122.172000i"),Object(l.b)("td",{parentName:"tr",align:null},"Science Registry value if available, otherwise the MaxMind City DB value")))),Object(l.b)("h3",{id:"value-fields"},"Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_bits"),Object(l.b)("td",{parentName:"tr",align:null},"939, 458, 560"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of bits in the (stitched) flow")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.num_packets"),Object(l.b)("td",{parentName:"tr",align:null},"77, 824"),Object(l.b)("td",{parentName:"tr",align:null},"Sum of the number of packets in the (stitched) flows")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.duration"),Object(l.b)("td",{parentName:"tr",align:null},"3.891"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as end minus start.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.bits_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"241, 443, 988"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_bits divided by duration")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.packets_per_second"),Object(l.b)("td",{parentName:"tr",align:null},"20, 001"),Object(l.b)("td",{parentName:"tr",align:null},"Calculated as num_packets divided by duration")))),Object(l.b)("h3",{id:"tstat-value-fields"},"Tstat Value Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_cwin_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_initial_cwin"),Object(l.b)("td",{parentName:"tr",align:null},"313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_max_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"64313")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_min_seg_size"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_mss"),Object(l.b)("td",{parentName:"tr",align:null},"8960")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_out_seq_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_dup"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fc"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_fs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_reor"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_rto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unfs"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unk"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_pkts_unrto"),Object(l.b)("td",{parentName:"tr",align:null},"0")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_bytes"),Object(l.b)("td",{parentName:"tr",align:null},"1678")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rexmit_pkts"),Object(l.b)("td",{parentName:"tr",align:null},"2")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_avg"),Object(l.b)("td",{parentName:"tr",align:null},"0.044")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_max"),Object(l.b)("td",{parentName:"tr",align:null},"39.527")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_min"),Object(l.b)("td",{parentName:"tr",align:null},"0.001")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_rtt_std"),Object(l.b)("td",{parentName:"tr",align:null},"0.276")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_sack_cnt"),Object(l.b)("td",{parentName:"tr",align:null},"1")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_max"),Object(l.b)("td",{parentName:"tr",align:null},"1549681")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_win_min"),Object(l.b)("td",{parentName:"tr",align:null},"17")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"values.tcp_window_scale"),Object(l.b)("td",{parentName:"tr",align:null},"13")))),Object(l.b)("h3",{id:"developer-fields"},"Developer Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@ingest_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 10:03:20.700"),Object(l.b)("td",{parentName:"tr",align:null},"Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@timestamp"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:21.703"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@exit_time"),Object(l.b)("td",{parentName:"tr",align:null},"Jun 9, 2020 @ 18:03:25.369"),Object(l.b)("td",{parentName:"tr",align:null},"The time the flow exited the pipeline")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@processing_time"),Object(l.b)("td",{parentName:"tr",align:null},"688.31"),Object(l.b)("td",{parentName:"tr",align:null},"@exit_time minus @ingest_time. Useful for seeing how long stitching took.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"stitched_flows"),Object(l.b)("td",{parentName:"tr",align:null},"13"),Object(l.b)("td",{parentName:"tr",align:null},"Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"es_doc_id"),Object(l.b)("td",{parentName:"tr",align:null},"4f46bef884..."),Object(l.b)("td",{parentName:"tr",align:null},"Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"tags"),Object(l.b)("td",{parentName:"tr",align:null},"maxmind src asn"),Object(l.b)("td",{parentName:"tr",align:null},"Various info and error messages")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"trial"),Object(l.b)("td",{parentName:"tr",align:null},"5"),Object(l.b)("td",{parentName:"tr",align:null},"Can be set in 40-aggregation.conf if desired")))),Object(l.b)("h3",{id:"elasticsearch-fields"},"Elasticsearch Fields"),Object(l.b)("table",null,Object(l.b)("thead",{parentName:"table"},Object(l.b)("tr",{parentName:"thead"},Object(l.b)("th",{parentName:"tr",align:null},"name"),Object(l.b)("th",{parentName:"tr",align:null},"example"),Object(l.b)("th",{parentName:"tr",align:null},"description"))),Object(l.b)("tbody",{parentName:"table"},Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_index"),Object(l.b)("td",{parentName:"tr",align:null},"om-ns-netsage-2020.06.14"),Object(l.b)("td",{parentName:"tr",align:null},'name of the index ("database table")')),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_type"),Object(l.b)("td",{parentName:"tr",align:null},"_doc"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_id"),Object(l.b)("td",{parentName:"tr",align:null},"HRkcm3IByJ9fEnbnCpaY"),Object(l.b)("td",{parentName:"tr",align:null},"elasticsearch document id. If es_doc_id is provided, that is used.")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"_score"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES query")),Object(l.b)("tr",{parentName:"tbody"},Object(l.b)("td",{parentName:"tr",align:null},"@version"),Object(l.b)("td",{parentName:"tr",align:null},"1"),Object(l.b)("td",{parentName:"tr",align:null},"set by ES")))))}s.isMDXComponent=!0},228:function(t,e,a){"use strict";a.d(e,"a",(function(){return p})),a.d(e,"b",(function(){return u}));var n=a(0),r=a.n(n);function l(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function b(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function i(t){for(var e=1;e<arguments.length;e++){var a=null!=arguments[e]?arguments[e]:{};e%2?b(Object(a),!0).forEach((function(e){l(t,e,a[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(a)):b(Object(a)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(a,e))}))}return t}function c(t,e){if(null==t)return{};var a,n,r=function(t,e){if(null==t)return{};var a,n,r={},l=Object.keys(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n<l.length;n++)a=l[n],e.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var d=r.a.createContext({}),s=function(t){var e=r.a.useContext(d),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},p=function(t){var e=s(t.components);return r.a.createElement(d.Provider,{value:e},t.children)},m={inlineCode:"code",wrapper:function(t){var e=t.children;return r.a.createElement(r.a.Fragment,{},e)}},o=r.a.forwardRef((function(t,e){var a=t.components,n=t.mdxType,l=t.originalType,b=t.parentName,d=c(t,["components","mdxType","originalType","parentName"]),p=s(a),o=n,u=p["".concat(b,".").concat(o)]||p[o]||m[o]||l;return a?r.a.createElement(u,i(i({ref:e},d),{},{components:a})):r.a.createElement(u,i({ref:e},d))}));function u(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var l=a.length,b=new Array(l);b[0]=o;var i={};for(var c in e)hasOwnProperty.call(e,c)&&(i[c]=e[c]);i.originalType=t,i.mdxType="string"==typeof t?t:n,b[1]=i;for(var d=2;d<l;d++)b[d]=a[d];return r.a.createElement.apply(null,b)}return r.a.createElement.apply(null,a)}o.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/c0bd86e8.4043049b.js b/assets/js/c0bd86e8.06ac168e.js
similarity index 96%
rename from assets/js/c0bd86e8.4043049b.js
rename to assets/js/c0bd86e8.06ac168e.js
index 6853ed9d..e10a06fc 100644
--- a/assets/js/c0bd86e8.4043049b.js
+++ b/assets/js/c0bd86e8.06ac168e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[117],{187:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"pipeline",title:"Pipeline",sidebar_label:"Intro"},l={unversionedId:"pipeline",id:"version-1.2.5/pipeline",isDocsHomePage:!1,title:"Pipeline",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.5/pipeline.md",slug:"/pipeline",permalink:"/netsage-pipeline/docs/1.2.5/pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline.md",version:"1.2.5",sidebar_label:"Intro",sidebar:"version-1.2.5/Pipeline",next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer"}},c=[{value:"Components",id:"components",children:[]},{value:"Sensors and Data Collection",id:"sensors-and-data-collection",children:[{value:"Importer",id:"importer",children:[]}]}],p={toc:c};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h1",{id:"description"},"Description"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc."),Object(o.b)("h2",{id:"components"},"Components"),Object(o.b)("p",null,"The Pipeline is made of the following components (Currently)"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"Importer"),"  (Collection of perl scripts)",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_importer"},"doc")))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Elastic Logstash")," Performs a variety of transformation on the data",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_logstash"},"doc")," "))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ")," used for message passing and queing of tasks.")),Object(o.b)("h2",{id:"sensors-and-data-collection"},"Sensors and Data Collection"),Object(o.b)("p",null,'"Testpoints" or "sensors" collect flow data (',Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). '),Object(o.b)("p",null,"Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files."),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return f}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),s=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},b=function(e){var t=s(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),b=s(n),d=r,f=b["".concat(a,".").concat(d)]||b[d]||u[d]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var p=2;p<o;p++)a[p]=n[p];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[114],{184:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"pipeline",title:"Pipeline",sidebar_label:"Intro"},l={unversionedId:"pipeline",id:"version-1.2.5/pipeline",isDocsHomePage:!1,title:"Pipeline",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.5/pipeline.md",slug:"/pipeline",permalink:"/netsage-pipeline/docs/1.2.5/pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline.md",version:"1.2.5",sidebar_label:"Intro",sidebar:"version-1.2.5/Pipeline",next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.5/pipeline_importer"}},c=[{value:"Components",id:"components",children:[]},{value:"Sensors and Data Collection",id:"sensors-and-data-collection",children:[{value:"Importer",id:"importer",children:[]}]}],p={toc:c};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h1",{id:"description"},"Description"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc."),Object(o.b)("h2",{id:"components"},"Components"),Object(o.b)("p",null,"The Pipeline is made of the following components (Currently)"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"Importer"),"  (Collection of perl scripts)",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_importer"},"doc")))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Elastic Logstash")," Performs a variety of transformation on the data",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_logstash"},"doc")," "))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ")," used for message passing and queing of tasks.")),Object(o.b)("h2",{id:"sensors-and-data-collection"},"Sensors and Data Collection"),Object(o.b)("p",null,'"Testpoints" or "sensors" collect flow data (',Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). '),Object(o.b)("p",null,"Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files."),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return f}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),s=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},b=function(e){var t=s(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),b=s(n),d=r,f=b["".concat(a,".").concat(d)]||b[d]||u[d]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var p=2;p<o;p++)a[p]=n[p];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/c3f49368.7fcf56c0.js b/assets/js/c3f49368.e29a1244.js
similarity index 97%
rename from assets/js/c3f49368.7fcf56c0.js
rename to assets/js/c3f49368.e29a1244.js
index ae4b8660..7ee5df9e 100644
--- a/assets/js/c3f49368.7fcf56c0.js
+++ b/assets/js/c3f49368.e29a1244.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[118],{188:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(232)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.9/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.9/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/intro.md",version:"1.2.9",sidebar_label:"Intro",sidebar:"version-1.2.9/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/pipeline/importer"},"Doc"),", ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},"Doc"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[115],{185:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),r=(n(0),n(228)),o={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-1.2.9/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.9/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/intro.md",version:"1.2.9",sidebar_label:"Intro",sidebar:"version-1.2.9/Pipeline",next:{title:"Tstat",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(r.b)("h2",{id:"description"},"Description"),Object(r.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(r.b)("h2",{id:"data-collection"},"Data Collection"),Object(r.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(r.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(r.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(r.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(r.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(r.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(r.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(r.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(r.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components (currently)"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/pipeline/importer"},"Doc"),", ",Object(r.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(r.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},"Doc"),")"),Object(r.b)("li",{parentName:"ul"},Object(r.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(r.b)("h2",{id:"visualization"},"Visualization"),Object(r.b)("p",null,Object(r.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(r.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(r.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Server" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the "Docker Simple" Install should suffice. The "Docker Advanced" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(o,".").concat(u)]||d[u]||b[u]||r;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,o=new Array(r);o[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return i.a.createElement.apply(null,o)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/c4f5d8e4.76411a79.js b/assets/js/c4f5d8e4.4e0e3f0d.js
similarity index 78%
rename from assets/js/c4f5d8e4.76411a79.js
rename to assets/js/c4f5d8e4.4e0e3f0d.js
index 93117486..ae8c0e6d 100644
--- a/assets/js/c4f5d8e4.76411a79.js
+++ b/assets/js/c4f5d8e4.4e0e3f0d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[119],{189:function(e,t,a){"use strict";a.r(t);var n=a(0),i=a.n(n),o=a(249),r=a.n(o),c=a(240),l=a(237),s=a(16),m=a(238),u=a(190),d=a.n(u);t.default=function(){var e=Object(s.default)().siteConfig,t=void 0===e?{}:e;return i.a.createElement(c.a,{title:""+t.title,description:"Description will go into a meta tag in <head />"},i.a.createElement("header",{className:r()("hero hero--primary",d.a.heroBanner)},i.a.createElement("div",{className:"container"},i.a.createElement("h1",{className:"hero__title"},t.title),i.a.createElement("img",{src:"img/netsage.webp"}),i.a.createElement("p",{className:"hero__subtitle"},t.tagline),i.a.createElement("div",{className:d.a.buttons},i.a.createElement(l.a,{className:r()("button button--outline button--secondary button--lg",d.a.getStarted),to:Object(m.a)("docs/pipeline/intro")},"Pipeline Documentation")))))}}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[116],{186:function(e,t,a){"use strict";a.r(t);var n=a(0),i=a.n(n),o=a(245),r=a.n(o),c=a(236),l=a(233),s=a(16),m=a(234),u=a(187),d=a.n(u);t.default=function(){var e=Object(s.default)().siteConfig,t=void 0===e?{}:e;return i.a.createElement(c.a,{title:""+t.title,description:"Description will go into a meta tag in <head />"},i.a.createElement("header",{className:r()("hero hero--primary",d.a.heroBanner)},i.a.createElement("div",{className:"container"},i.a.createElement("h1",{className:"hero__title"},t.title),i.a.createElement("img",{src:"img/netsage.webp"}),i.a.createElement("p",{className:"hero__subtitle"},t.tagline),i.a.createElement("div",{className:d.a.buttons},i.a.createElement(l.a,{className:r()("button button--outline button--secondary button--lg",d.a.getStarted),to:Object(m.a)("docs/pipeline/intro")},"Pipeline Documentation")))))}}}]);
\ No newline at end of file
diff --git a/assets/js/c51f4c72.0e8bec24.js b/assets/js/c51f4c72.69c062f8.js
similarity index 99%
rename from assets/js/c51f4c72.0e8bec24.js
rename to assets/js/c51f4c72.69c062f8.js
index 63796730..e2ad3c94 100644
--- a/assets/js/c51f4c72.0e8bec24.js
+++ b/assets/js/c51f4c72.69c062f8.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[120],{191:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.9/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",version:"1.2.9",sidebar_label:"Docker Simple",sidebar:"version-1.2.9/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}]},{value:"Upgrading",id:"upgrading",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)"),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_name\n")),Object(r.b)("p",null,'Replace "tag_name" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h3",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("p",null,Object(r.b)("h3",{parentName:"p"},"Shut things down"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(r.b)("p",{parentName:"p"},"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(r.b)("h3",{parentName:"p"},"Update Source Code"),Object(r.b)("p",{parentName:"p"},"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(r.b)("p",{parentName:"div"},"Example:\n",Object(r.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(r.b)("h3",{parentName:"p"},"Check/Update Files"),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(r.b)("h3",{parentName:"p"},"Select Release Version"),Object(r.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b tag_value \ngit pull\n./scripts/docker_select_version.sh\n")),Object(r.b)("p",{parentName:"p"},"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(r.b)("h3",{parentName:"p"},"Update Docker Containers"),Object(r.b)("p",{parentName:"p"},"Do not forget this step!  This applies for both development and release versions."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(r.b)("h3",{parentName:"p"},"Restart Docker Containers"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(r.b)("h3",{parentName:"p"},"Delete old images"),Object(r.b)("p",{parentName:"p"},"To save space, delete any old images that are not needed. ")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),m=a,h=d["".concat(i,".").concat(m)]||d[m]||b[m]||r;return n?o.a.createElement(h,l(l({ref:t},c),{},{components:n})):o.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[117],{188:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Default Installation Guide",sidebar_label:"Docker Simple"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.9/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Default Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",version:"1.2.9",sidebar_label:"Docker Simple",sidebar:"version-1.2.9/Pipeline",previous:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"},next:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}]},{value:"Upgrading",id:"upgrading",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)"),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout tag_name\n")),Object(r.b)("p",null,'Replace "tag_name" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  "),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "rtr.one.rndnet.edu" and "rtr.two.nrdnet.edu". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name)."),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h3",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))),Object(r.b)("h2",{id:"upgrading"},"Upgrading"),Object(r.b)("p",null,Object(r.b)("h3",{parentName:"p"},"Shut things down"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(r.b)("p",{parentName:"p"},"This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(r.b)("h3",{parentName:"p"},"Update Source Code"),Object(r.b)("p",{parentName:"p"},"To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing."),Object(r.b)("p",{parentName:"div"},"Example:\n",Object(r.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(r.b)("h3",{parentName:"p"},"Check/Update Files"),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: "3.7" at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},'In the same files, see if the version of nfdump has changed. Look for lines like "image: netsage/nfdump-collector:1.6.18". If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)\nNote that you do not need to update the versions of the importer or logstash images. That will be done for you in the "select release version" stop coming up.')),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.")),Object(r.b)("li",{parentName:"ul"},Object(r.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely."))),Object(r.b)("h3",{parentName:"p"},"Select Release Version"),Object(r.b)("p",{parentName:"p"},'Run these two commands to select the new release you want to run. In the first, replace "tag_value" by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.'),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b tag_value \ngit pull\n./scripts/docker_select_version.sh\n")),Object(r.b)("p",{parentName:"p"},"Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  "),Object(r.b)("h3",{parentName:"p"},"Update Docker Containers"),Object(r.b)("p",{parentName:"p"},"Do not forget this step!  This applies for both development and release versions."),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(r.b)("h3",{parentName:"p"},"Restart Docker Containers"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(r.b)("h3",{parentName:"p"},"Delete old images"),Object(r.b)("p",{parentName:"p"},"To save space, delete any old images that are not needed. ")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),m=a,h=d["".concat(i,".").concat(m)]||d[m]||b[m]||r;return n?o.a.createElement(h,l(l({ref:t},c),{},{components:n})):o.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/cbd1c0d0.5208c323.js b/assets/js/cbd1c0d0.5208c323.js
deleted file mode 100644
index 7ab1d0c8..00000000
--- a/assets/js/cbd1c0d0.5208c323.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[121],{192:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(232)),l={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},i={unversionedId:"deploy/docker_install_advanced",id:"deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/docs/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_advanced.md",version:"current",sidebar_label:"Docker Advanced Options",sidebar:"Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Filter Flows by Subnet",id:"to-filter-flows-by-subnet",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-filter-flows-by-subnet"},"To Filter Flows by Subnet"),Object(r.b)("p",null,'With this option, flows from specified sensors will be dropped unless src or dst is in the list of subnets to keep. It works similarly to the option to filter by interface.  "ALL" can refer to all sensors.\nIf a sensor is not referenced at all, all of its flows will be kept. '),Object(r.b)("p",null,"For example,"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"subnet_filter_flag=True\nsubnet_filter_keep=Sensor A Name: 123.45.6.0/16; Sensor B Name: 123.33.33.0/24, 456.66.66.0/24\n")),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(l,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,i(i({ref:t},c),{},{components:n})):a.a.createElement(h,i({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,l=new Array(r);l[0]=u;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:o,l[1]=i;for(var c=2;c<r;c++)l[c]=n[c];return a.a.createElement.apply(null,l)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/cbd1c0d0.6ea2feaf.js b/assets/js/cbd1c0d0.6ea2feaf.js
new file mode 100644
index 00000000..05e32054
--- /dev/null
+++ b/assets/js/cbd1c0d0.6ea2feaf.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[118],{189:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/docs/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_advanced.md",version:"current",sidebar_label:"Docker Advanced Options",sidebar:"Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Save Flow Data to a Different Location",id:"to-save-flow-data-to-a-different-location",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h4",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"image: copy from the default collector sections already in the file. "),Object(r.b)("li",{parentName:"ul"},'command: choose between "sfcapd" for sflow and "nfcapd" for netflow, and at the end of the command, specify the port to watch for incoming flow data.  '),Object(r.b)("li",{parentName:"ul"},'volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we\'re writing to ./data/input_data/example. Change "example" to something meaningful.'),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) ")),Object(r.b)("p",null,"Make sure the indentation is right or you'll get an error about yaml parsing."),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h4",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a new "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; don\'t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate. (Enter "netflow" if you\'re running IPFIX.)'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h4",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=MyNet Los Angeles sFlow\n")),Object(r.b)("h4",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. "),Object(r.b)("p",null,"In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure ",Object(r.b)("inlineCode",{parentName:"p"},"ifindex_filter_flag=True"),' with "True" capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_keep=123\nifindex_filter_keep=123,456\nifindex_filter_keep=Sensor 1: 789\nifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900\n")),Object(r.b)("p",null,"In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)\nIn the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.\nIn the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.\nIn the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  "),Object(r.b)("p",null,"Spaces don't matter except within the sensor names. Punctuation is required as shown."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk doesn't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>1</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-save-flow-data-to-a-different-location"},"To Save Flow Data to a Different Location"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options."),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.")),Object(r.b)("p",null,"During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline dir}\nmkdir /var/netsage\nrm data/.placeholder\nrmdir data\nln -s /var/netsage {netsage-pipeline dir}/data\n")),Object(r.b)("p",null,"(Check the permissions of the directory.)"),Object(r.b)("ol",{start:2},Object(r.b)("li",{parentName:"ol"},"Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir/input_data/netflow:/data")," (similarly for sflow) and set the importer and logstash volumes to ",Object(r.b)("inlineCode",{parentName:"li"},"- /mydir:/data"),". ")),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8."),Object(r.b)("p",null,"To do this, edit LS_JAVA_OPTS in the .env file. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"LS_JAVA_OPTS=-Xmx4g -Xms4g\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.")),Object(r.b)("p",null,"To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),u=o,h=p["".concat(i,".").concat(u)]||p[u]||m[u]||r;return n?a.a.createElement(h,l(l({ref:t},c),{},{components:n})):a.a.createElement(h,l({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/cd902920.a8ebc09f.js b/assets/js/cd902920.0ae4e688.js
similarity index 97%
rename from assets/js/cd902920.a8ebc09f.js
rename to assets/js/cd902920.0ae4e688.js
index 1b7b80fd..a5831b88 100644
--- a/assets/js/cd902920.a8ebc09f.js
+++ b/assets/js/cd902920.0ae4e688.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[122],{193:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(232)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.10/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",version:"1.2.10",sidebar_label:"Troubleshooting",sidebar:"version-1.2.10/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(a.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(a.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Logstash or Importer errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure all containers are running.  ",Object(a.b)("inlineCode",{parentName:"li"},"docker ps")),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(a.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(a.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(a.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are comsuming. You need to add more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. ")),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(a.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),d=n,f=p["".concat(i,".").concat(d)]||p[d]||b[d]||a;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,i=new Array(a);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c<a;c++)i[c]=o[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,o)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[119],{190:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(228)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.10/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",version:"1.2.10",sidebar_label:"Troubleshooting",sidebar:"version-1.2.10/Pipeline",previous:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"If you are not seeing flows after installation",id:"if-you-are-not-seeing-flows-after-installation",children:[]},{value:"If flow collection stops",id:"if-flow-collection-stops",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"if-you-are-not-seeing-flows-after-installation"},"If you are not seeing flows after installation"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(a.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(a.b)("li",{parentName:"ul"},"Use ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(a.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(a.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(a.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(a.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(a.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(a.b)("h3",{id:"if-flow-collection-stops"},"If flow collection stops"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Logstash or Importer errors:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure all containers are running.  ",Object(a.b)("inlineCode",{parentName:"li"},"docker ps")),Object(a.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(a.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(a.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully.")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Disk space:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: ",Object(a.b)("inlineCode",{parentName:"li"},"docker image prune -a")," and ",Object(a.b)("inlineCode",{parentName:"li"},"docker container prune"),"."),Object(a.b)("li",{parentName:"ul"},"Also check to see how much space the nfcapd files are comsuming. You need to add more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). ")),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Memory:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"If you are running a lot of data, sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. ")),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(a.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return f}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},d=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),d=n,f=p["".concat(i,".").concat(d)]||p[d]||b[d]||a;return o?r.a.createElement(f,l(l({ref:t},c),{},{components:o})):r.a.createElement(f,l({ref:t},c))}));function f(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,i=new Array(a);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c<a;c++)i[c]=o[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,o)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ceb45c47.a3330b5f.js b/assets/js/ceb45c47.a3330b5f.js
deleted file mode 100644
index c07857a3..00000000
--- a/assets/js/ceb45c47.a3330b5f.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[123],{194:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/docs/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_upgrade.md",version:"current",sidebar_label:"Docker - Upgrading",sidebar:"Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,'Run these three commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ceb45c47.e6acb003.js b/assets/js/ceb45c47.e6acb003.js
new file mode 100644
index 00000000..e8dbc4ab
--- /dev/null
+++ b/assets/js/ceb45c47.e6acb003.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[120],{191:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},l={unversionedId:"deploy/docker_upgrade",id:"deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/docs/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/next/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_upgrade.md",version:"current",sidebar_label:"Docker - Upgrading",sidebar:"Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"}},c=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],d={toc:c};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},d,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit fetch\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,"Run these commands to upgrade to the current announced version. While older versions of the pipeline are available, they are not recommended or supported for daily use. If in doubt, the Master branch contains the latest version. The documentation version at the top of this page is also the current pipeline version number."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git switch {tag} \ngit pull\n./Upgrade_x_x_x.sh (where x_x_x is the current version you are upgrading to)\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var d=o.a.createContext({}),s=function(e){var t=o.a.useContext(d),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=s(e.components);return o.a.createElement(d.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,d=c(e,["components","mdxType","originalType","parentName"]),p=s(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,l(l({ref:t},d),{},{components:n})):o.a.createElement(m,l({ref:t},d))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var d=2;d<a;d++)i[d]=n[d];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d1597237.705b0ef9.js b/assets/js/d1597237.07f1898e.js
similarity index 95%
rename from assets/js/d1597237.705b0ef9.js
rename to assets/js/d1597237.07f1898e.js
index 3c7ce285..63b1f6ac 100644
--- a/assets/js/d1597237.705b0ef9.js
+++ b/assets/js/d1597237.07f1898e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[124],{195:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(232)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.9/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.9/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_pipeline.md",version:"1.2.9"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[121],{192:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return c})),t.d(n,"metadata",(function(){return i})),t.d(n,"toc",(function(){return p})),t.d(n,"default",(function(){return l}));var r=t(3),o=t(7),a=(t(0),t(228)),c={},i={unversionedId:"components/docker_pipeline",id:"version-1.2.9/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (all containers) using:",source:"@site/versioned_docs/version-1.2.9/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.9/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_pipeline.md",version:"1.2.9"},p=[],s={toc:p};function l(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,t,{components:n,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Start up the pipeline (all containers) using:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(a.b)("p",null,'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(a.b)("p",null,"You can see the status of the containers and whether any have died (exited) using"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(a.b)("p",null,"To check the logs for each of the containers, run"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(a.b)("p",null,"Add ",Object(a.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(a.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(a.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(a.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(a.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(a.b)("p",null,"To shut down the pipeline (all containers) use"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}l.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return m}));var r=t(0),o=t.n(r);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function c(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?c(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):c(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function p(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)t=a[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var s=o.a.createContext({}),l=function(e){var n=o.a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},u=function(e){var n=l(e.components);return o.a.createElement(s.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},b=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),u=l(t),b=r,m=u["".concat(c,".").concat(b)]||u[b]||d[b]||a;return t?o.a.createElement(m,i(i({ref:n},s),{},{components:t})):o.a.createElement(m,i({ref:n},s))}));function m(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,c=new Array(a);c[0]=b;var i={};for(var p in n)hasOwnProperty.call(n,p)&&(i[p]=n[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var s=2;s<a;s++)c[s]=t[s];return o.a.createElement.apply(null,c)}return o.a.createElement.apply(null,t)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d1b2eded.fdea14ce.js b/assets/js/d1b2eded.fdea14ce.js
deleted file mode 100644
index 7e1344ae..00000000
--- a/assets/js/d1b2eded.fdea14ce.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[125],{196:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return c}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"version-2.0.0/pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/versioned_docs/version-2.0.0/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/importer.md",version:"2.0.0",sidebar_label:"Importer",sidebar:"version-2.0.0/Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],s={toc:l};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=i.a.createContext({}),c=function(e){var t=i.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=c(e.components);return i.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},s),{},{components:n})):i.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var s=2;s<o;s++)a[s]=n[s];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d2c255b4.51d1c434.js b/assets/js/d2c255b4.51d1c434.js
deleted file mode 100644
index b951a643..00000000
--- a/assets/js/d2c255b4.51d1c434.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[126],{197:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"choose_install",title:"Choosing an Installation Procedure",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-2.0.0/deploy/choose_install",isDocsHomePage:!1,title:"Choosing an Installation Procedure",description:"Manual or BareMetal Installation",source:"@site/versioned_docs/version-2.0.0/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/choosing.md",version:"2.0.0",sidebar_label:"Choose Install",sidebar:"version-2.0.0/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/pipeline/elastic"},next:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/deploy/bare_metal_install"}},s=[{value:"Manual or BareMetal Installation",id:"manual-or-baremetal-installation",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"manual-or-baremetal-installation"},"Manual or BareMetal Installation"),Object(a.b)("p",null,"The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(a.b)("p",null,"It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up."),Object(a.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(a.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(a.b)("p",null,"The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(a.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(a.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"bare_metal_install"},"Manual/Server Installation")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(a.b)("li",{parentName:"ul"},Object(a.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/deploy/docker_install_advanced"},"Advanced Docker")," - options that allow for more complex configurations")))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d2c4f39e.b83f8cc2.js b/assets/js/d2c4f39e.d6c2c200.js
similarity index 96%
rename from assets/js/d2c4f39e.b83f8cc2.js
rename to assets/js/d2c4f39e.d6c2c200.js
index d283929d..381d8057 100644
--- a/assets/js/d2c4f39e.b83f8cc2.js
+++ b/assets/js/d2c4f39e.d6c2c200.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[127],{198:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(232)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.8/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.8/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/choosing.md",version:"1.2.8",sidebar_label:"Choose Install",sidebar:"version-1.2.8/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[122],{193:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.8/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.8/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.8/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/choosing.md",version:"1.2.8",sidebar_label:"Choose Install",sidebar:"version-1.2.8/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.8/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d3556f45.a168af5e.js b/assets/js/d3556f45.2aeec99a.js
similarity index 95%
rename from assets/js/d3556f45.a168af5e.js
rename to assets/js/d3556f45.2aeec99a.js
index 9d4af9a9..6f4370d0 100644
--- a/assets/js/d3556f45.a168af5e.js
+++ b/assets/js/d3556f45.2aeec99a.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[128],{199:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/docs/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/importer.md",version:"current",sidebar_label:"Importer",sidebar:"Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/next/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],c={toc:l};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=s(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},c),{},{components:n})):i.a.createElement(m,p({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var c=2;c<o;c++)a[c]=n[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[123],{194:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"importer",title:"Importer",sidebar_label:"Importer"},p={unversionedId:"pipeline/importer",id:"pipeline/importer",isDocsHomePage:!1,title:"Importer",description:'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsagedeidentifierraw" RabbitMQ queue.',source:"@site/docs/pipeline/importer.md",slug:"/pipeline/importer",permalink:"/netsage-pipeline/docs/next/pipeline/importer",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/importer.md",version:"current",sidebar_label:"Importer",sidebar:"Pipeline",previous:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump"},next:{title:"Logstash Pipeline",permalink:"/netsage-pipeline/docs/next/pipeline/logstash"}},l=[{value:"Configuration",id:"configuration",children:[]}],c={toc:l};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,'A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the "netsage_deidentifier_raw" RabbitMQ queue.\nAll flow data waits in the queue until it is read in and processed by the logstash pipeline.'),Object(o.b)("p",null,'To read nfcapd files, the importer uses an nfdump command with the "-a" option to aggregate raw flows within the file by the "5-tuple," i.e., the source and destination IPs, ports, and protocol. The  "-L" option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. '),Object(o.b)("h3",{id:"configuration"},"Configuration"),Object(o.b)("p",null,"Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs."),Object(o.b)("p",null,"To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. "))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),s=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},u=function(e){var t=s(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=s(n),d=r,m=u["".concat(a,".").concat(d)]||u[d]||f[d]||o;return n?i.a.createElement(m,p(p({ref:t},c),{},{components:n})):i.a.createElement(m,p({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var c=2;c<o;c++)a[c]=n[c];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d48b255e.d6a84f1e.js b/assets/js/d48b255e.1c5329df.js
similarity index 98%
rename from assets/js/d48b255e.d6a84f1e.js
rename to assets/js/d48b255e.1c5329df.js
index 519a09b6..ac1ba0c9 100644
--- a/assets/js/d48b255e.d6a84f1e.js
+++ b/assets/js/d48b255e.1c5329df.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[129],{200:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"version-1.2.11/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",version:"1.2.11",sidebar_label:"Docker - Upgrading",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,'Run these three commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the conctainers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[124],{195:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"version-1.2.11/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",version:"1.2.11",sidebar_label:"Docker - Upgrading",sidebar:"version-1.2.11/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,'Run these three commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the conctainers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d5087a67.4b6f0656.js b/assets/js/d5087a67.cb6a7230.js
similarity index 98%
rename from assets/js/d5087a67.4b6f0656.js
rename to assets/js/d5087a67.cb6a7230.js
index 22fe8741..38803ceb 100644
--- a/assets/js/d5087a67.4b6f0656.js
+++ b/assets/js/d5087a67.cb6a7230.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[130],{201:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.10","label":"1.2.10","isLast":false,"docsSidebars":{"version-1.2.10/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.10/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/1.2.10/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.10/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.10/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.10/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},{"type":"link","label":"Docker Upgrading","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/1.2.10/devel/docusaurus"},{"type":"link","label":"Taggin a Release","href":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/choose_install":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/dev_dataset":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docusaurus":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/elastic":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/importer":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/intro":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/logstash":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/nfdump":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/tstat":"version-1.2.10/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[125],{196:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.10","label":"1.2.10","isLast":false,"docsSidebars":{"version-1.2.10/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.10/pipeline/intro"},{"type":"link","label":"Tstat Data","href":"/netsage-pipeline/docs/1.2.10/pipeline/tstat"},{"type":"link","label":"Sflow/Netflow Data","href":"/netsage-pipeline/docs/1.2.10/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.10/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.10/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.10/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.10/deploy/choose_install"},{"type":"link","label":"Manual Installation","href":"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"},{"type":"link","label":"Docker Installation","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced Options","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"},{"type":"link","label":"Docker Upgrading","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Development","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.10/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"},{"type":"link","label":"Docusaurus","href":"/netsage-pipeline/docs/1.2.10/devel/docusaurus"},{"type":"link","label":"Taggin a Release","href":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/choose_install":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/dev_dataset":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/devel/docusaurus":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/elastic":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/importer":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/intro":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/logstash":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/nfdump":"version-1.2.10/Pipeline","/netsage-pipeline/docs/1.2.10/pipeline/tstat":"version-1.2.10/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/d5140579.770e5402.js b/assets/js/d5140579.0f850372.js
similarity index 97%
rename from assets/js/d5140579.770e5402.js
rename to assets/js/d5140579.0f850372.js
index f1935e9a..a79c3f78 100644
--- a/assets/js/d5140579.770e5402.js
+++ b/assets/js/d5140579.0f850372.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[131],{202:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return s})),o.d(t,"toc",(function(){return l})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(232)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},s={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.8/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",version:"1.2.8",sidebar_label:"Troubleshooting",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"}},l=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]}],c={toc:l};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(i.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(i.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(i.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(i.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(i.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. "),Object(i.b)("p",null,"Please see the ",Object(i.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(i.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(i.b)("p",null,"Alternatively you may also try doing this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(i.b)("p",null,"Reference: ",Object(i.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return m}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function s(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function l(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):s(s({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=u(o),b=n,m=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return o?r.a.createElement(m,s(s({ref:t},c),{},{components:o})):r.a.createElement(m,s({ref:t},c))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:n,a[1]=s;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[126],{197:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return a})),o.d(t,"metadata",(function(){return s})),o.d(t,"toc",(function(){return l})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),i=(o(0),o(228)),a={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},s={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.8/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",version:"1.2.8",sidebar_label:"Troubleshooting",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset"}},l=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]}],c={toc:l};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(i.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(i.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(i.b)("p",null,Object(i.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 "),Object(i.b)("li",{parentName:"ul"},"Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed."),Object(i.b)("li",{parentName:"ul"},"Use ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose ps")," to be sure the collectors (and other containers) are running."),Object(i.b)("li",{parentName:"ul"},'Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  "nfdump -r filename" will show the flows in a file.'),Object(i.b)("li",{parentName:"ul"},"Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data."),Object(i.b)("li",{parentName:"ul"},"Check the logs of the various containers to see if anything jumps out as being invalid.\xa0 ",Object(i.b)("inlineCode",{parentName:"li"},"docker-compose logs -f $service_label")),Object(i.b)("li",{parentName:"ul"},"Check the logs to see if logstash is starting successfully. "),Object(i.b)("li",{parentName:"ul"},"If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.")),Object(i.b)("p",null,"To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  ",Object(i.b)("inlineCode",{parentName:"p"},"http://localhost:15672")," in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published."),Object(i.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(i.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 2GB of RAM. You'll need to update the jvm.options file to grant it more memory. "),Object(i.b)("p",null,"Please see the ",Object(i.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced#customize-logstash-settings"},"Docker Advanced guide")," for details on how to customize logstash."),Object(i.b)("p",null,"Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(i.b)("p",null,"Alternatively you may also try doing this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(i.b)("p",null,"Reference: ",Object(i.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return m}));var n=o(0),r=o.n(n);function i(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function s(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?a(Object(o),!0).forEach((function(t){i(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):a(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function l(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)o=i[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):s(s({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,i=e.originalType,a=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),p=u(o),b=n,m=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return o?r.a.createElement(m,s(s({ref:t},c),{},{components:o})):r.a.createElement(m,s({ref:t},c))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=o.length,a=new Array(i);a[0]=b;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:n,a[1]=s;for(var c=2;c<i;c++)a[c]=o[c];return r.a.createElement.apply(null,a)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d5479f42.d126a913.js b/assets/js/d5479f42.b8876961.js
similarity index 96%
rename from assets/js/d5479f42.d126a913.js
rename to assets/js/d5479f42.b8876961.js
index 9d1962ab..af5e1976 100644
--- a/assets/js/d5479f42.d126a913.js
+++ b/assets/js/d5479f42.b8876961.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[132],{203:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),i=(n(0),n(232)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.6/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal",source:"@site/versioned_docs/version-1.2.6/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/choosing.md",version:"1.2.6",sidebar_label:"Choose Install",sidebar:"version-1.2.6/Pipeline",previous:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker"},next:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install"}},s=[{value:"BareMetal",id:"baremetal",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal"},"BareMetal"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"docker_install"},"Docker")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"install"},"BareMetal"))))}u.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,f=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[127],{198:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return u}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.6/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal",source:"@site/versioned_docs/version-1.2.6/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/choosing.md",version:"1.2.6",sidebar_label:"Choose Install",sidebar:"version-1.2.6/Pipeline",previous:{title:"Docker Guide",permalink:"/netsage-pipeline/docs/1.2.6/devel/docker"},next:{title:"NetSage Flow Processing Pipeline Install Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install"}},s=[{value:"BareMetal",id:"baremetal",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function u(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal"},"BareMetal"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"docker_install"},"Docker")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"install"},"BareMetal"))))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),u=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=u(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(n),b=r,f=p["".concat(a,".").concat(b)]||p[b]||d[b]||i;return n?o.a.createElement(f,l(l({ref:t},c),{},{components:n})):o.a.createElement(f,l({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d71ff009.7f914194.js b/assets/js/d71ff009.037347f4.js
similarity index 96%
rename from assets/js/d71ff009.7f914194.js
rename to assets/js/d71ff009.037347f4.js
index cbc52db6..7a49ee09 100644
--- a/assets/js/d71ff009.7f914194.js
+++ b/assets/js/d71ff009.037347f4.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[133],{204:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(232)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.7/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",version:"1.2.7",sidebar_label:"Troubleshooting",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(a.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(a.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(a.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(a.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 1GB of RAM. You'll need to update the jvm.options file to grant it more memory."),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced#customize-logstash-settings"},"advaned section")," for details on how to customize logstash"),Object(a.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(a.b)("p",null,"You may also try the pattern below, if that still isn't enough."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},232:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return m}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),b=n,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return o?r.a.createElement(m,l(l({ref:t},c),{},{components:o})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c<a;c++)i[c]=o[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[128],{199:function(e,t,o){"use strict";o.r(t),o.d(t,"frontMatter",(function(){return i})),o.d(t,"metadata",(function(){return l})),o.d(t,"toc",(function(){return s})),o.d(t,"default",(function(){return u}));var n=o(3),r=o(7),a=(o(0),o(228)),i={id:"docker_troubleshoot",title:"Docker Troubleshooting",sidebar_label:"Troubleshooting"},l={unversionedId:"deploy/docker_troubleshoot",id:"version-1.2.7/deploy/docker_troubleshoot",isDocsHomePage:!1,title:"Docker Troubleshooting",description:"Troubleshooting",source:"@site/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",slug:"/deploy/docker_troubleshoot",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",version:"1.2.7",sidebar_label:"Troubleshooting",sidebar:"version-1.2.7/Pipeline",previous:{title:"Docker Advanced Installation Guide",permalink:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"},next:{title:"Pipeline Replay Dataset",permalink:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset"}},s=[{value:"Troubleshooting",id:"troubleshooting",children:[{value:"Data Flow issues:",id:"data-flow-issues",children:[]},{value:"Resource Limitations",id:"resource-limitations",children:[]}]}],c={toc:s};function u(e){var t=e.components,o=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(n.a)({},c,o,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h2",{id:"troubleshooting"},"Troubleshooting"),Object(a.b)("h3",{id:"data-flow-issues"},"Data Flow issues:"),Object(a.b)("p",null,Object(a.b)("strong",{parentName:"p"},"Troubleshooting checklist:")),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Make sure you configured your routers to point to the correct address/port where the collector is running.\xa0 hostname:9999 is the default."),Object(a.b)("li",{parentName:"ul"},"Make sure you created a .env file and updated the settings accordingly."),Object(a.b)("li",{parentName:"ul"},"sensorName especially since that identifies the source of the data."),Object(a.b)("li",{parentName:"ul"},"check the logs of the various components to see if anything jumps out as being invalid.\xa0 docker-compose logs -f <service_label>")),Object(a.b)("h3",{id:"resource-limitations"},"Resource Limitations"),Object(a.b)("p",null,"If you are running a lot of data sometimes docker may need to be allocated more memory. The most\nlikely culprit is logstash which is usually only allocated 1GB of RAM. You'll need to update the jvm.options file to grant it more memory."),Object(a.b)("p",null,"Please see the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced#customize-logstash-settings"},"advaned section")," for details on how to customize logstash"),Object(a.b)("p",null,"Applying this snippet to logstash may help. Naturally the values will have to change."),Object(a.b)("p",null,"You may also try the pattern below, if that still isn't enough."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},"environment: + LS_JAVA_OPTS=-Xmx3g\n")),Object(a.b)("p",null,"Alternatively you may also try doing this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-yaml"},'deploy:\n  resources:\n    limits:\n      cpus: "0.50"\n      memory: 50M\n    reservations:\n      cpus: "0.25"\n      memory: 20M\n')),Object(a.b)("p",null,"Reference: ",Object(a.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/compose-file/#resources"},"https://docs.docker.com/compose/compose-file/#resources")))}u.isMDXComponent=!0},228:function(e,t,o){"use strict";o.d(t,"a",(function(){return p})),o.d(t,"b",(function(){return m}));var n=o(0),r=o.n(n);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function i(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function l(e){for(var t=1;t<arguments.length;t++){var o=null!=arguments[t]?arguments[t]:{};t%2?i(Object(o),!0).forEach((function(t){a(e,t,o[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(o)):i(Object(o)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(o,t))}))}return e}function s(e,t){if(null==e)return{};var o,n,r=function(e,t){if(null==e)return{};var o,n,r={},a=Object.keys(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)o=a[n],t.indexOf(o)>=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var c=r.a.createContext({}),u=function(e){var t=r.a.useContext(c),o=t;return e&&(o="function"==typeof e?e(t):l(l({},t),e)),o},p=function(e){var t=u(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=u(o),b=n,m=p["".concat(i,".").concat(b)]||p[b]||d[b]||a;return o?r.a.createElement(m,l(l({ref:t},c),{},{components:o})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=o.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c<a;c++)i[c]=o[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,o)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d74c151a.f23bb09f.js b/assets/js/d74c151a.f23bb09f.js
new file mode 100644
index 00000000..ffe139ec
--- /dev/null
+++ b/assets/js/d74c151a.f23bb09f.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[129],{200:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return u}));var o=n(3),a=n(7),i=(n(0),n(228)),r={id:"docusaurus",title:"Revising Documentation",sidebar_label:"Docusaurus"},s={unversionedId:"devel/docusaurus",id:"version-2.1.0/devel/docusaurus",isDocsHomePage:!1,title:"Revising Documentation",description:"This project's documentation uses Docusaurus.",source:"@site/versioned_docs/version-2.1.0/devel/documentation_guide.md",slug:"/devel/docusaurus",permalink:"/netsage-pipeline/docs/devel/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/documentation_guide.md",version:"2.1.0",sidebar_label:"Docusaurus",sidebar:"version-2.1.0/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide"},next:{title:"How to Release a New Version of the Pipeline",permalink:"/netsage-pipeline/docs/devel/docker_dev_tag"}},c=[{value:"If Not Using Docker",id:"if-not-using-docker",children:[{value:"Installation",id:"installation",children:[]},{value:"If Local Development",id:"if-local-development",children:[]},{value:"To Make Changes",id:"to-make-changes",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio",children:[]},{value:"Removing a version",id:"removing-a-version",children:[]}]},{value:"If Using Docker",id:"if-using-docker",children:[{value:"Build and Start the Container",id:"build-and-start-the-container",children:[]},{value:"To Make Changes",id:"to-make-changes-1",children:[]},{value:"Tagging a New release",id:"tagging-a-new-release-1",children:[]},{value:"Deploying Docs to github.io",id:"deploying-docs-to-githubio-1",children:[]}]}],l={toc:c};function u(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(o.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This project's documentation uses Docusaurus."),Object(i.b)("p",null,"Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver."),Object(i.b)("p",null,"Yarn is a package manager for JavaScript and replaces the\xa0npm\xa0client. It is not strictly necessary but highly encouraged."),Object(i.b)("p",null,"To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation\nat the ",Object(i.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(i.b)("p",null,Object(i.b)("em",{parentName:"p"},"THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.")),Object(i.b)("h2",{id:"if-not-using-docker"},"If Not Using Docker"),Object(i.b)("p",null,"These are instructions for editing and releasing docs without using Docker."),Object(i.b)("h3",{id:"installation"},"Installation"),Object(i.b)("p",null,"To get started the first time, install npm, then use that to install yarn "),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ sudo yum install npm\n$ sudo npm install -g yarn \n")),Object(i.b)("p",null,"Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ yarn install\n")),Object(i.b)("h3",{id:"if-local-development"},"If Local Development"),Object(i.b)("p",null,"If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the\xa0build\xa0directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn build  \n$ yarn start\ngo to http://localhost:3000\n")),Object(i.b)("h3",{id:"to-make-changes"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("p",null,'To view the changes you\'ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the "Deploying Docs to github.io" section below.'),Object(i.b)("h3",{id:"tagging-a-new-release"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ yarn run docusaurus docs:version a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio"},"Deploying Docs to github.io"),Object(i.b)("p",null,'Whether you have created a new set of versioned tags or just want to update the docs in "master", to make changes appear at ',Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io/netsage-pipeline"},"https://netsage-project.github.io/netsage-pipeline"),", do the following."),Object(i.b)("p",null,"If Travis or some other CI is working, it will run yarn install and yarn deploy to do this automatically."),Object(i.b)("p",null,"If it is not, do it manually:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'$ USE_SSH="true" GIT_USER="your-username" yarn deploy   \n')),Object(i.b)("p",null,"replacing your-username.  This sets a couple env vars then runs 'yarn deploy' which runs 'docusaurus deploy' (see package.json) which pushes the static website created to url: \"",Object(i.b)("a",{parentName:"p",href:"https://netsage-project.github.io%22"},'https://netsage-project.github.io"')," (see docusaurus.config.js) "),Object(i.b)("p",null,"NOTE: You need to have created ssh keys on the host you are running this on and added them to your github account. "),Object(i.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(i.b)("p",null,"To remove version 1.2.6 of the docs, for example,"),Object(i.b)("p",null,"we need to: "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(i.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(i.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json"),Object(i.b)("li",{parentName:"ul"},"change 1.2.6 in docusaurus.config.js back to 1.2.5")),Object(i.b)("p",null,"Then git add, commit, and push"),Object(i.b)("h2",{id:"if-using-docker"},"If Using Docker"),Object(i.b)("p",null,"You may also use a docs Docker container to simplify installation, making changes, and deployment.  This method starts a local web server that allows you to see changes to the docs in a browser on your local machine, as they are made."),Object(i.b)("h3",{id:"build-and-start-the-container"},"Build and Start the Container"),Object(i.b)("p",null,"Git clone the netsage pipeline project then build and start the container.\nThe Dockerfile in website/ tells how to build an image that runs yarn.  Docker-compose.yml brings up a docs container."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ cd netsage-pipeline/website\n$ docker-compose build build_docs\n$ docker-compose up -d docs\ngo to http://localhost:8000/netsage-pipeline/\n")),Object(i.b)("h3",{id:"to-make-changes-1"},"To Make Changes"),Object(i.b)("p",null,"Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.\nWhen finished, git add, git commit, git push, as usual.\nRepeat as needed."),Object(i.b)("h3",{id:"tagging-a-new-release-1"},"Tagging a New release"),Object(i.b)("p",null,"When it's time to release a new version of the Pipeline, you need to create a new version of the docs as well. "),Object(i.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"$ docker-compose build build_docs\n$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c\n")),Object(i.b)("p",null,"replacing a.b.c with the next release version number.",Object(i.b)("br",{parentName:"p"}),"\n","This will create new versioned docs in website/versioned_docs/."),Object(i.b)("p",null,"Then edit docusaurus.config.js and change ",Object(i.b)("inlineCode",{parentName:"p"},"lastVersion:")," to refer to the new version number. "),Object(i.b)("p",null,"Finally, commit and push the following to github:"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"website/versioned_docs/version-a.b.c/"),Object(i.b)("li",{parentName:"ul"},"website/versioned_sidebars/version-a.b.c.sidebars.json"),Object(i.b)("li",{parentName:"ul"},"versions.json "),Object(i.b)("li",{parentName:"ul"},"docusaurus.config.js")),Object(i.b)("h3",{id:"deploying-docs-to-githubio-1"},"Deploying Docs to github.io"),Object(i.b)("p",null,"How to do this when using Docker ??? Get into the container ???"),Object(i.b)("p",null,"For now, go a linux server that has yarn installed and\nfollow the instructions under If Not Using Docker."))}u.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return h}));var o=n(0),a=n.n(o);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},i=Object.keys(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o<i.length;o++)n=i[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),u=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},d=function(e){var t=u(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,r=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),d=u(n),p=o,h=d["".concat(r,".").concat(p)]||d[p]||b[p]||i;return n?a.a.createElement(h,s(s({ref:t},l),{},{components:n})):a.a.createElement(h,s({ref:t},l))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=p;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:o,r[1]=s;for(var l=2;l<i;l++)r[l]=n[l];return a.a.createElement.apply(null,r)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/d75898ab.84f0767e.js b/assets/js/d75898ab.8089cc99.js
similarity index 97%
rename from assets/js/d75898ab.84f0767e.js
rename to assets/js/d75898ab.8089cc99.js
index 33534fdf..08e8b52f 100644
--- a/assets/js/d75898ab.84f0767e.js
+++ b/assets/js/d75898ab.8089cc99.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[134],{205:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.9","label":"1.2.9","isLast":false,"docsSidebars":{"version-1.2.9/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.9/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.9/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.9/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.9/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.9/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.9/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/choose_install":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/devel/dev_dataset":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/docusaurus":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/elastic":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/importer":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/intro":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/logstash":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/nfdump":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/tstat":"version-1.2.9/Pipeline"}}')}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[130],{201:function(e){e.exports=JSON.parse('{"pluginId":"default","version":"1.2.9","label":"1.2.9","isLast":false,"docsSidebars":{"version-1.2.9/Pipeline":[{"collapsed":true,"type":"category","label":"Pipeline","items":[{"type":"link","label":"Intro","href":"/netsage-pipeline/docs/1.2.9/pipeline/intro"},{"type":"link","label":"Tstat","href":"/netsage-pipeline/docs/1.2.9/pipeline/tstat"},{"type":"link","label":"Nfdump","href":"/netsage-pipeline/docs/1.2.9/pipeline/nfdump"},{"type":"link","label":"Importer","href":"/netsage-pipeline/docs/1.2.9/pipeline/importer"},{"type":"link","label":"Logstash","href":"/netsage-pipeline/docs/1.2.9/pipeline/logstash"},{"type":"link","label":"Elasticsearch","href":"/netsage-pipeline/docs/1.2.9/pipeline/elastic"}]},{"collapsed":true,"type":"category","label":"Deployment","items":[{"type":"link","label":"Choose Install","href":"/netsage-pipeline/docs/1.2.9/deploy/choose_install"},{"type":"link","label":"Server Installation Guide","href":"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"},{"type":"link","label":"Docker Simple","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},{"type":"link","label":"Docker Advanced","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},{"type":"link","label":"Troubleshooting","href":"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"}]},{"collapsed":true,"type":"category","label":"Deployment Guides","items":[{"type":"link","label":"Replay Dataset","href":"/netsage-pipeline/docs/1.2.9/devel/dev_dataset"},{"type":"link","label":"Docker Dev Guide","href":"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}]},{"collapsed":true,"type":"category","label":"Documentation","items":[{"type":"link","label":"Documentation Guide","href":"/netsage-pipeline/docs/1.2.9/docusaurus"}]}]},"permalinkToSidebar":{"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/choose_install":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/devel/dev_dataset":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/docusaurus":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/elastic":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/importer":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/intro":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/logstash":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/nfdump":"version-1.2.9/Pipeline","/netsage-pipeline/docs/1.2.9/pipeline/tstat":"version-1.2.9/Pipeline"}}')}}]);
\ No newline at end of file
diff --git a/assets/js/d91f9def.b4c78c77.js b/assets/js/d91f9def.f3256090.js
similarity index 95%
rename from assets/js/d91f9def.b4c78c77.js
rename to assets/js/d91f9def.f3256090.js
index 6cfbe6f5..bb63bef6 100644
--- a/assets/js/d91f9def.b4c78c77.js
+++ b/assets/js/d91f9def.f3256090.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[135],{206:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return u})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.6/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.",source:"@site/versioned_docs/version-1.2.6/DOCUMENTATION.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.6/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/DOCUMENTATION.md",version:"1.2.6",sidebar_label:"Documentation Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}},u=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]}],l={toc:u};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  "),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h3",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h3",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h3",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function u(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),s=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=u(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,m=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var u in t)hasOwnProperty.call(t,u)&&(c[u]=t[u]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[131],{202:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return u})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.6/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.",source:"@site/versioned_docs/version-1.2.6/DOCUMENTATION.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.6/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/DOCUMENTATION.md",version:"1.2.6",sidebar_label:"Documentation Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}},u=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]}],l={toc:u};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  "),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h3",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h3",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h3",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function u(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),s=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,l=u(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,m=d["".concat(i,".").concat(b)]||d[b]||p[b]||a;return n?o.a.createElement(m,c(c({ref:t},l),{},{components:n})):o.a.createElement(m,c({ref:t},l))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var u in t)hasOwnProperty.call(t,u)&&(c[u]=t[u]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var l=2;l<a;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/db0ed613.02dc005f.js b/assets/js/db0ed613.c9712562.js
similarity index 97%
rename from assets/js/db0ed613.02dc005f.js
rename to assets/js/db0ed613.c9712562.js
index 15e14e33..564d46a7 100644
--- a/assets/js/db0ed613.02dc005f.js
+++ b/assets/js/db0ed613.c9712562.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[136],{207:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.9/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.9/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.9/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/documentation_guide.md",version:"1.2.9",sidebar_label:"Documentation Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[132],{203:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"docusaurus",title:"Documentation Guide",sidebar_label:"Documentation Guide"},c={unversionedId:"docusaurus",id:"version-1.2.9/docusaurus",isDocsHomePage:!1,title:"Documentation Guide",description:"Using Local Tools",source:"@site/versioned_docs/version-1.2.9/documentation_guide.md",slug:"/docusaurus",permalink:"/netsage-pipeline/docs/1.2.9/docusaurus",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/documentation_guide.md",version:"1.2.9",sidebar_label:"Documentation Guide",sidebar:"version-1.2.9/Pipeline",previous:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"}},l=[{value:"Installation",id:"installation",children:[]},{value:"Local Development",id:"local-development",children:[]},{value:"Build",id:"build",children:[]},{value:"Build the container",id:"build-the-container",children:[]},{value:"Local Development",id:"local-development-1",children:[{value:"Removing a version",id:"removing-a-version",children:[]}]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h1",{id:"using-local-tools"},"Using Local Tools"),Object(a.b)("p",null,"This project's documentation is using docusaurus to generate docs from markdown files and loaded in a react app."),Object(a.b)("p",null,"To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation\nat the ",Object(a.b)("a",{parentName:"p",href:"https://v2.docusaurus.io/"},"docusaurus 2")," project website."),Object(a.b)("h2",{id:"installation"},"Installation"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn\n")),Object(a.b)("h2",{id:"local-development"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn start\n")),Object(a.b)("p",null,"This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server."),Object(a.b)("h2",{id:"build"},"Build"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ yarn build\n")),Object(a.b)("p",null,"This command generates static content into the ",Object(a.b)("inlineCode",{parentName:"p"},"build")," directory and can be served using any static contents hosting service."),Object(a.b)("h1",{id:"using-docker"},"Using Docker"),Object(a.b)("h2",{id:"build-the-container"},"Build the container"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose build build_docs\n")),Object(a.b)("h2",{id:"local-development-1"},"Local Development"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compopse up -d docs\n")),Object(a.b)("p",null,"The docs will be visible at http://localhost:8000/netsage-pipeline/"),Object(a.b)("h1",{id:"tagging-new-release"},"Tagging New release"),Object(a.b)("p",null,"Once the documentation is stable and you don't forsee any new change, please do the following:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"yarn run docusaurus docs:version 1.1.0\n")),Object(a.b)("p",null,"Where 1.1.0 is the next release version.  Commit the new directories created under "),Object(a.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies."),Object(a.b)("pre",{parentName:"div"},Object(a.b)("code",{parentName:"pre"},"docker-compose run  docs yarn run docusaurus docs:version  1.1.0\n")))),Object(a.b)("p",null,"(note if image is not built yet, please run: ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose build build_docs"),")"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"website/versioned_docs"),Object(a.b)("li",{parentName:"ul"},"website/versioned_sidebars"),Object(a.b)("li",{parentName:"ul"},"versions.json ")),Object(a.b)("p",null,"Update the docusaurus.config.js"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"lastVersion: '1.2.6' \n")),Object(a.b)("p",null,"should point to the latest value."),Object(a.b)("h3",{id:"removing-a-version"},"Removing a version"),Object(a.b)("p",null,"To remove version 1.2.6 for example."),Object(a.b)("p",null,"we need to: "),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"update versions.json to remove the reference"),Object(a.b)("li",{parentName:"ul"},"remove the versioned_docs/version-1.2.6"),Object(a.b)("li",{parentName:"ul"},"remove versioned_sidebars/version-1.2.6-sidebars.json")))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},p=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=d(n),p=r,m=u["".concat(i,".").concat(p)]||u[p]||b[p]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=p;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/dd1a9f26.51ca4c1a.js b/assets/js/dd1a9f26.a545ed72.js
similarity index 99%
rename from assets/js/dd1a9f26.51ca4c1a.js
rename to assets/js/dd1a9f26.a545ed72.js
index d0cd2ce2..f101e2be 100644
--- a/assets/js/dd1a9f26.51ca4c1a.js
+++ b/assets/js/dd1a9f26.a545ed72.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[137],{208:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.11/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",version:"1.2.11",sidebar_label:"Docker Installation",sidebar:"version-1.2.11/Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[133],{204:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.11/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",version:"1.2.11",sidebar_label:"Docker Installation",sidebar:"version-1.2.11/Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Set up a Pipeline Host",id:"2-set-up-a-pipeline-host",children:[]},{value:"3. Clone the Netsage Pipeline Project",id:"3-clone-the-netsage-pipeline-project",children:[]},{value:"4. Create Docker-compose.override.yml",id:"4-create-docker-composeoverrideyml",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"6. Create Environment File",id:"6-create-environment-file",children:[]},{value:"Testing the Collectors",id:"testing-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a ",Object(r.b)("em",{parentName:"p"},"sensor")," (i.e., flow ",Object(r.b)("em",{parentName:"p"},"exporter")," / router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there will be ",Object(r.b)("em",{parentName:"p"},"collectors")," (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-set-up-a-pipeline-host"},"2. Set up a Pipeline Host"),Object(r.b)("p",null,"Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. "),Object(r.b)("p",null,"Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/engine/install/"},"https://docs.docker.com/engine/install/"),"."),Object(r.b)("p",null,"Install Docker Compose from Docker's GitHub repository - see ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"https://docs.docker.com/compose/install/"),".  You need to ",Object(r.b)("strong",{parentName:"p"},"specify version 1.29.2")," (or newer) in the curl command. "),Object(r.b)("p",null,"Check default file permissions. If the ",Object(r.b)("em",{parentName:"p"},"logstash")," user is not able to access the logstash config files in the git checkout, you'll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work."),Object(r.b)("h3",{id:"3-clone-the-netsage-pipeline-project"},"3. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"Clone the netsage-pipeline project from github."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version."),Object(r.b)("p",null,"Move into the netsage-pipeline/ directory (",Object(r.b)("strong",{parentName:"p"},"all git and docker commands must be run from inside this directory!"),"), then checkout the most recent version of the code. It will say you are in 'detached HEAD' state if you don't include -b."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.11".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.11."),Object(r.b)("h3",{id:"4-create-docker-composeoverrideyml"},"4. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. "),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("li",{parentName:"ul"},'If the collectors need to listen to different ports, make the appropriate changes here in both the "command:" and "ports:" lines. '),Object(r.b)("li",{parentName:"ul"},"By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in ",Object(r.b)("inlineCode",{parentName:"li"},"./data/input_data/")," (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.")),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem."))),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which image versions Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier. "),Object(r.b)("p",null,"This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values."),Object(r.b)("h3",{id:"6-create-environment-file"},"6. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Next, copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# Importer settings\nsflowSensorName=My sflow sensor name\nnetflowSensorName=My netflow sensor name\n")),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "MyNet New York Sflow" and "MyNet Boston Netflow" or "MyNet New York - London" and "MyNet New York - Paris". Whatever makes sense in your situation.'))),Object(r.b)("p",{parentName:"p"},"You will also want to edit the ",Object(r.b)("strong",{parentName:"p"},"Logstash output rabbit queue")," section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"rabbitmq_output_host=rabbit@mynet.edu\nrabbitmq_output_username=guest\nrabbitmq_output_pw=guest\nrabbitmq_output_key=netsage_archive_input\n")),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)"))),Object(r.b)("p",{parentName:"p"},"The following options are described in the Docker Advanced section:"),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To drop all flows except those using the specfied interfaces"),": Use if only some flows from a router are of interest and those can be identified by interface."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"To change the sensor name for flows using a certain interface"),": Use if you want to break out some flows coming into a port and give them a different sensor name."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},'To "manually" correct flow sizes and rates for sampling for specified sensors'),": Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.")),Object(r.b)("h2",{id:"testing-the-collectors"},"Testing the Collectors"),Object(r.b)("p",null,"At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  "),Object(r.b)("p",null,"(See the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"Subdirectories for sflow/netflow, year, month, and day are created automatically under ",Object(r.b)("inlineCode",{parentName:"p"},"data/input_data/"),". File names contain dates and times.\nThese are not text files; to view the contents, use an ",Object(r.b)("a",{parentName:"p",href:"http://www.linuxcertif.com/man/1/nfdump/"},"nfdump command")," (you will need to install nfdump).\nFiles will be deleted automatically by the importer as they age out (the default is to keep 3 days).  "),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)",Object(r.b)("br",{parentName:"p"}),"\n","See Troubleshooting if you have problems."),Object(r.b)("p",null,"To stop the collectors"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down \n")),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose logs\n# docker-compose logs logstash\n# docker-compose logs importer\netc.\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"# docker-compose down\n")),Object(r.b)("p",{parentName:"p"},"Run all commands from the netsage-pipeline/ directory.")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},u=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),u=o,m=d["".concat(i,".").concat(u)]||d[u]||b[u]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ded423ee.df3ed6d9.js b/assets/js/ded423ee.df3ed6d9.js
deleted file mode 100644
index 9531ce4a..00000000
--- a/assets/js/ded423ee.df3ed6d9.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[138],{209:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-2.0.0/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-2.0.0/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/pipeline_dataset.md",version:"2.0.0",sidebar_label:"Replay Dataset",sidebar:"version-2.0.0/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e00e17a0.481c92d6.js b/assets/js/e00e17a0.8c02aab3.js
similarity index 96%
rename from assets/js/e00e17a0.481c92d6.js
rename to assets/js/e00e17a0.8c02aab3.js
index 2358390f..a8507316 100644
--- a/assets/js/e00e17a0.481c92d6.js
+++ b/assets/js/e00e17a0.8c02aab3.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[139],{210:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.8/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",version:"1.2.8",sidebar_label:"Replay Dataset",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[134],{205:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.8/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",version:"1.2.8",sidebar_label:"Replay Dataset",sidebar:"version-1.2.8/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e10970d8.6486e99c.js b/assets/js/e10970d8.cde1bc28.js
similarity index 97%
rename from assets/js/e10970d8.6486e99c.js
rename to assets/js/e10970d8.cde1bc28.js
index c8f96bff..f6f764dd 100644
--- a/assets/js/e10970d8.6486e99c.js
+++ b/assets/js/e10970d8.cde1bc28.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[140],{211:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(232)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.10/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.10/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_env.md",version:"1.2.10"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "RNDNet New York - London 1" and "RNDNet New York - London 2". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). (For NetSage, another logstash pipeline on a remote server moves flows from this final rabbit queue into Elasticsearch.)"),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[135],{206:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),r=(n(0),n(228)),i={},s={unversionedId:"components/docker_env",id:"version-1.2.10/components/docker_env",isDocsHomePage:!1,title:"docker_env",description:"Please copy env.example to .env",source:"@site/versioned_docs/version-1.2.10/components/docker_env.md",slug:"/components/docker_env",permalink:"/netsage-pipeline/docs/1.2.10/components/docker_env",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_env.md",version:"1.2.10"},c=[],l={toc:c};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(r.b)("wrapper",Object(a.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",null,"then edit the .env file to set the sensor names"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",null,"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "RNDNet New York - London 1" and "RNDNet New York - London 2". Whatever makes sense in your situation.'))),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",null,"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). (For NetSage, another logstash pipeline on a remote server moves flows from this final rabbit queue into Elasticsearch.)"),Object(r.b)("p",null,"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",null,Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return m})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=o.a.createContext({}),p=function(e){var t=o.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},m=function(e){var t=p(e.components);return o.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},d=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,i=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),m=p(n),d=a,b=m["".concat(i,".").concat(d)]||m[d]||u[d]||r;return n?o.a.createElement(b,s(s({ref:t},l),{},{components:n})):o.a.createElement(b,s({ref:t},l))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e186625e.3f1e3436.js b/assets/js/e186625e.6c1b54b7.js
similarity index 95%
rename from assets/js/e186625e.3f1e3436.js
rename to assets/js/e186625e.6c1b54b7.js
index 966cf123..6c903463 100644
--- a/assets/js/e186625e.3f1e3436.js
+++ b/assets/js/e186625e.6c1b54b7.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[141],{212:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.7/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.7/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/tstat.md",version:"1.2.7",sidebar_label:"Tstat",sidebar:"version-1.2.7/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[136],{207:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat",sidebar_label:"Tstat"},s={unversionedId:"pipeline/tstat",id:"version-1.2.7/pipeline/tstat",isDocsHomePage:!1,title:"Tstat",description:"Netsage Usage",source:"@site/versioned_docs/version-1.2.7/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/tstat.md",version:"1.2.7",sidebar_label:"Tstat",sidebar:"version-1.2.7/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/intro"},next:{title:"Nfdump",permalink:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump"}},p=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker",id:"docker",children:[]}],c={toc:p};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage project ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," provides a way to send the captured data to a rabbitmq host where it can then be processed by a ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),l=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},b=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=l(n),b=r,f=u["".concat(o,".").concat(b)]||u[b]||d[b]||i;return n?a.a.createElement(f,s(s({ref:t},c),{},{components:n})):a.a.createElement(f,s({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=b;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var c=2;c<i;c++)o[c]=n[c];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e2be698d.6ab0c41b.js b/assets/js/e2be698d.dd0e2d10.js
similarity index 99%
rename from assets/js/e2be698d.6ab0c41b.js
rename to assets/js/e2be698d.dd0e2d10.js
index 78ec01cf..8f60a89f 100644
--- a/assets/js/e2be698d.6ab0c41b.js
+++ b/assets/js/e2be698d.dd0e2d10.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[142],{213:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return p}));var o=n(3),i=n(7),a=(n(0),n(232)),s={id:"install",title:"NetSage Flow Processing Pipeline Install Guide",sidebar_label:"Installation Guide"},r={unversionedId:"deploy/install",id:"version-1.2.6/deploy/install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Install Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.6/deploy/install.md",slug:"/deploy/install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/install.md",version:"1.2.6",sidebar_label:"Installation Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}},l=[{value:"Components",id:"components",children:[]},{value:"Prerequirements",id:"prerequirements",children:[{value:"nfdump",id:"nfdump",children:[]}]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"RabbitMQ",id:"rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline",id:"installing-the-pipeline",children:[]},{value:"Configuration",id:"configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Configuring the Pipeline Stages",id:"configuring-the-pipeline-stages",children:[]},{value:"Shared config file listing",id:"shared-config-file-listing",children:[]}]},{value:"Running the daemons",id:"running-the-daemons",children:[{value:"Daemon Listing",id:"daemon-listing",children:[]}]},{value:"Setup Notes",id:"setup-notes",children:[]}],c={toc:l};function p(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(a.b)("h2",{id:"components"},"Components"),Object(a.b)("p",null,"Minimum components"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Data Injestion Source (nfdump, tstat or both)"),Object(a.b)("li",{parentName:"ul"},"RabbitMQ"),Object(a.b)("li",{parentName:"ul"},"LogStash"),Object(a.b)("li",{parentName:"ul"},"Importer (If you use nfdump)")),Object(a.b)("h2",{id:"prerequirements"},"Prerequirements"),Object(a.b)("p",null,"The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed."),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"nfdump "),Object(a.b)("li",{parentName:"ol"},"tstat")),Object(a.b)("p",null,"You'll need to have at least one of them setup in order to be able to process the data"),Object(a.b)("h3",{id:"nfdump"},"nfdump"),Object(a.b)("p",null,"The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is ",Object(a.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(a.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(a.b)("p",null,"Once nfdump is setup you'll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. "),Object(a.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(a.b)("h3",{id:"rabbitmq"},"RabbitMQ"),Object(a.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n`\n\n")),Object(a.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service\n")),Object(a.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(a.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(a.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(a.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(a.b)("p",null,"For Red Hat/CentOS 6, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(a.b)("p",null,"For Red Hat/CentOS 7, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(a.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(a.b)("h2",{id:"installing-the-pipeline"},"Installing the Pipeline"),Object(a.b)("p",null,"Pipeline components:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise."),Object(a.b)("li",{parentName:"ol"},"Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them")),Object(a.b)("p",null,"send directly to the appropriate rabbit queue. "),Object(a.b)("ol",{start:3},Object(a.b)("li",{parentName:"ol"},"Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows ")),Object(a.b)("p",null,"that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.\nThe final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the\nlast logstash conf here to write to elasticsearch."),Object(a.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. Install it like this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(a.b)("h2",{id:"configuration"},"Configuration"),Object(a.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(a.b)("p",null,"Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files ","[THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]",". This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")," "),Object(a.b)("p",null,"The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'<collection>\n    <flow-path>/path/to/flow-files</flow-path>\n    <sensor>hostname.tld</sensor>\n\x3c!--  "instance" goes along with sensor\n       This is to identify various instances if a sensor has more than one "stream" / data collection\n       Defaults to 0.\n    <instance>1</instance>\n--\x3e\n  \x3c!--\n       Defaults to sensor, but you can set it to something else here\n      <router-address></router-address>\n  --\x3e\n \x3c!--\n    Flow type: type of flow data (defaults to netflow)\n --\x3e\n \x3c!--\n    <flow-type>sflow</flow-type>\n --\x3e\n</collection>\n')),Object(a.b)("p",null,"Notice that ",Object(a.b)("inlineCode",{parentName:"p"},"instance")," , ",Object(a.b)("inlineCode",{parentName:"p"},"router-address")," , and ",Object(a.b)("inlineCode",{parentName:"p"},"flow-type")," are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file."),Object(a.b)("p",null,"You can have multiple ",Object(a.b)("inlineCode",{parentName:"p"},"collection")," stanzas, to import multiple collections on one host."),Object(a.b)("p",null,"The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"<config>\n  <collection>\n    <flow-path>/path/to/flow-files1</flow-path>\n    <sensor>hostname1.tld</sensor>\n  </collection>\n  <collection>\n    <flow-path>/path/to/flow-files2</flow-path>\n    <sensor>hostname2.tld</sensor>\n  </collection>\n\n  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_input>\n\n  \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_output>\n</config>\n")),Object(a.b)("h3",{id:"configuring-the-pipeline-stages"},"Configuring the Pipeline Stages"),Object(a.b)("p",null,"Each stage must be configured with Rabbit input and output queue information. The intention here is that flows should be deidentified before they leave the original node the flow data is collected on. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(a.b)("p",null,'The username/password are both set to "guest" by default, as this is the default provided by RabbitMQ. This works fine if the localhost is processing all the data. The configs look something like this (some have additional sections).'),Object(a.b)("p",null,"Notice that the only Rabbit connection information that's provided here is that which is not specific in the shared config file. This way if we need to change the credentials throughout the entire pipeline, it's easy to do."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-xml"},"   <config>\n     \x3c!-- rabbitmq connection info --\x3e\n     <rabbit_input>\n       <queue>netsage_deidentifier_raw</queue>\n       <channel>2</channel>\n     </rabbit_input>\n   \n     \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n     <rabbit_output>\n       <queue>netsage_deidentifier_cached</queue>\n       <channel>3</channel>\n     </rabbit_output>\n     <worker>\n         \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n         \x3c!-- for stitching, we can only use 1 --\x3e\n       <num-processes>1</num-processes>\n   \n       \x3c!-- where should we write the cache worker pid file to --\x3e\n       <pid-file>/var/run/netsage-cache-workers.pid</pid-file>\n   \n     </worker>\n   </config>\n")),Object(a.b)("p",null,"The defaults should work unless the pipeline stages need to be reordered for some reason, or if SSL or different hosts/credentials are needed. However, the very endpoints should be checked. At the moment that means the flow cache (which is the first stage in the pipeline) and the flow mover (the last stage)."),Object(a.b)("h3",{id:"shared-config-file-listing"},"Shared config file listing"),Object(a.b)("p",null,"The shared configuration files and logging configuration files are listed below (all of the pipeline components use these):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information\n/etc/grnoc/netsage/deidentifier/logging.conf - logging config\n/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled\n")),Object(a.b)("h2",{id:"running-the-daemons"},"Running the daemons"),Object(a.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(a.b)("p",null,"For more details on each individual daemon, use the ",Object(a.b)("inlineCode",{parentName:"p"},"--help")," flag."),Object(a.b)("h3",{id:"daemon-listing"},"Daemon Listing"),Object(a.b)("h4",{id:"netsage-netflow-importer-daemon"},"netsage-netflow-importer-daemon"),Object(a.b)("p",null,"This is a daemon that reads raw netflow data, reads it, and pushes it to a Rabbit queue for processing."),Object(a.b)("p",null,"Config file: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")," "),Object(a.b)("h2",{id:"setup-notes"},"Setup Notes"),Object(a.b)("p",null,"INPUT AND OUTPUT LOGSTASH FILTERS\nStandard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs (01-inputs.conf and 99-outputs.conf) may be modified for your use.\nTo use the provided 01-inputs.conf and 99-outputs.conf versions, fill in the IP of the final rabbit host in 99-outputs.conf, and put the rabbitmq usernames and passwords into the logstash keystore.\nYour 01 and 99 conf files should not be overwritten by upgrades."),Object(a.b)("p",null,"To set up the keystore:  (note that logstash-keystore takes a minute to come back with a prompt)\nBe sure /usr/share/logstash/config exists\n(the full path, in case you need it: /usr/share/logstash/bin/logstash-keystore)\nCreate logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create\n    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)\n```sh\nTo list the keys:\n")),Object(a.b)("p",null,"  $ sudo -E logstash-keystore list"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"To remove a key-value pair:\n\n```sh\n  $ sudo -E logstash-keystore remove <key name>\n")),Object(a.b)("p",null,'FLOW STITCHING - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use "-w 1".'),Object(a.b)("p",null,"See the comments in 04-stitching.conf to learn more about how complete flows are defined."))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),i=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,i=function(e,t){if(null==e)return{};var n,o,i={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),p=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},h=i.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(s,".").concat(h)]||u[h]||d[h]||a;return n?i.a.createElement(b,r(r({ref:t},c),{},{components:n})):i.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:o,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return i.a.createElement.apply(null,s)}return i.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[137],{208:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return p}));var o=n(3),i=n(7),a=(n(0),n(228)),s={id:"install",title:"NetSage Flow Processing Pipeline Install Guide",sidebar_label:"Installation Guide"},r={unversionedId:"deploy/install",id:"version-1.2.6/deploy/install",isDocsHomePage:!1,title:"NetSage Flow Processing Pipeline Install Guide",description:"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.6/deploy/install.md",slug:"/deploy/install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/install.md",version:"1.2.6",sidebar_label:"Installation Guide",sidebar:"version-1.2.6/Pipeline",previous:{title:"Choosing Install",permalink:"/netsage-pipeline/docs/1.2.6/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.6/deploy/docker_install"}},l=[{value:"Components",id:"components",children:[]},{value:"Prerequirements",id:"prerequirements",children:[{value:"nfdump",id:"nfdump",children:[]}]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"RabbitMQ",id:"rabbitmq",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline",id:"installing-the-pipeline",children:[]},{value:"Configuration",id:"configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Configuring the Pipeline Stages",id:"configuring-the-pipeline-stages",children:[]},{value:"Shared config file listing",id:"shared-config-file-listing",children:[]}]},{value:"Running the daemons",id:"running-the-daemons",children:[{value:"Daemon Listing",id:"daemon-listing",children:[]}]},{value:"Setup Notes",id:"setup-notes",children:[]}],c={toc:l};function p(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(a.b)("h2",{id:"components"},"Components"),Object(a.b)("p",null,"Minimum components"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"Data Injestion Source (nfdump, tstat or both)"),Object(a.b)("li",{parentName:"ul"},"RabbitMQ"),Object(a.b)("li",{parentName:"ul"},"LogStash"),Object(a.b)("li",{parentName:"ul"},"Importer (If you use nfdump)")),Object(a.b)("h2",{id:"prerequirements"},"Prerequirements"),Object(a.b)("p",null,"The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed."),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"nfdump "),Object(a.b)("li",{parentName:"ol"},"tstat")),Object(a.b)("p",null,"You'll need to have at least one of them setup in order to be able to process the data"),Object(a.b)("h3",{id:"nfdump"},"nfdump"),Object(a.b)("p",null,"The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is ",Object(a.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(a.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump."),Object(a.b)("p",null,"Once nfdump is setup you'll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. "),Object(a.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(a.b)("h3",{id:"rabbitmq"},"RabbitMQ"),Object(a.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n`\n\n")),Object(a.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service\n")),Object(a.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(a.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(a.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(a.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(a.b)("p",null,"For Red Hat/CentOS 6, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(a.b)("p",null,"For Red Hat/CentOS 7, create ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(a.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(a.b)("h2",{id:"installing-the-pipeline"},"Installing the Pipeline"),Object(a.b)("p",null,"Pipeline components:"),Object(a.b)("ol",null,Object(a.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise."),Object(a.b)("li",{parentName:"ol"},"Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them")),Object(a.b)("p",null,"send directly to the appropriate rabbit queue. "),Object(a.b)("ol",{start:3},Object(a.b)("li",{parentName:"ol"},"Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows ")),Object(a.b)("p",null,"that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.\nThe final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the\nlast logstash conf here to write to elasticsearch."),Object(a.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. Install it like this:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(a.b)("h2",{id:"configuration"},"Configuration"),Object(a.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(a.b)("p",null,"Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files ","[THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]",". This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")," "),Object(a.b)("p",null,"The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},'<collection>\n    <flow-path>/path/to/flow-files</flow-path>\n    <sensor>hostname.tld</sensor>\n\x3c!--  "instance" goes along with sensor\n       This is to identify various instances if a sensor has more than one "stream" / data collection\n       Defaults to 0.\n    <instance>1</instance>\n--\x3e\n  \x3c!--\n       Defaults to sensor, but you can set it to something else here\n      <router-address></router-address>\n  --\x3e\n \x3c!--\n    Flow type: type of flow data (defaults to netflow)\n --\x3e\n \x3c!--\n    <flow-type>sflow</flow-type>\n --\x3e\n</collection>\n')),Object(a.b)("p",null,"Notice that ",Object(a.b)("inlineCode",{parentName:"p"},"instance")," , ",Object(a.b)("inlineCode",{parentName:"p"},"router-address")," , and ",Object(a.b)("inlineCode",{parentName:"p"},"flow-type")," are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file."),Object(a.b)("p",null,"You can have multiple ",Object(a.b)("inlineCode",{parentName:"p"},"collection")," stanzas, to import multiple collections on one host."),Object(a.b)("p",null,"The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below)."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"<config>\n  <collection>\n    <flow-path>/path/to/flow-files1</flow-path>\n    <sensor>hostname1.tld</sensor>\n  </collection>\n  <collection>\n    <flow-path>/path/to/flow-files2</flow-path>\n    <sensor>hostname2.tld</sensor>\n  </collection>\n\n  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_input>\n\n  \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5671</port>\n    <username>guest</username>\n    <password>guest</password>\n    <batch_size>100</batch_size>\n    <vhost>netsage</vhost>\n    <ssl>0</ssl>\n    <cacert>/path/to/cert.crt</cacert> \x3c!-- required if ssl is 1 --\x3e\n  </rabbit_output>\n</config>\n")),Object(a.b)("h3",{id:"configuring-the-pipeline-stages"},"Configuring the Pipeline Stages"),Object(a.b)("p",null,"Each stage must be configured with Rabbit input and output queue information. The intention here is that flows should be deidentified before they leave the original node the flow data is collected on. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(a.b)("p",null,'The username/password are both set to "guest" by default, as this is the default provided by RabbitMQ. This works fine if the localhost is processing all the data. The configs look something like this (some have additional sections).'),Object(a.b)("p",null,"Notice that the only Rabbit connection information that's provided here is that which is not specific in the shared config file. This way if we need to change the credentials throughout the entire pipeline, it's easy to do."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-xml"},"   <config>\n     \x3c!-- rabbitmq connection info --\x3e\n     <rabbit_input>\n       <queue>netsage_deidentifier_raw</queue>\n       <channel>2</channel>\n     </rabbit_input>\n   \n     \x3c!-- The cache does not output to a rabbit queue (shared memory instead) but we still need something here --\x3e\n     <rabbit_output>\n       <queue>netsage_deidentifier_cached</queue>\n       <channel>3</channel>\n     </rabbit_output>\n     <worker>\n         \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n         \x3c!-- for stitching, we can only use 1 --\x3e\n       <num-processes>1</num-processes>\n   \n       \x3c!-- where should we write the cache worker pid file to --\x3e\n       <pid-file>/var/run/netsage-cache-workers.pid</pid-file>\n   \n     </worker>\n   </config>\n")),Object(a.b)("p",null,"The defaults should work unless the pipeline stages need to be reordered for some reason, or if SSL or different hosts/credentials are needed. However, the very endpoints should be checked. At the moment that means the flow cache (which is the first stage in the pipeline) and the flow mover (the last stage)."),Object(a.b)("h3",{id:"shared-config-file-listing"},"Shared config file listing"),Object(a.b)("p",null,"The shared configuration files and logging configuration files are listed below (all of the pipeline components use these):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information\n/etc/grnoc/netsage/deidentifier/logging.conf - logging config\n/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled\n")),Object(a.b)("h2",{id:"running-the-daemons"},"Running the daemons"),Object(a.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(a.b)("p",null,Object(a.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(a.b)("p",null,"For more details on each individual daemon, use the ",Object(a.b)("inlineCode",{parentName:"p"},"--help")," flag."),Object(a.b)("h3",{id:"daemon-listing"},"Daemon Listing"),Object(a.b)("h4",{id:"netsage-netflow-importer-daemon"},"netsage-netflow-importer-daemon"),Object(a.b)("p",null,"This is a daemon that reads raw netflow data, reads it, and pushes it to a Rabbit queue for processing."),Object(a.b)("p",null,"Config file: ",Object(a.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")," "),Object(a.b)("h2",{id:"setup-notes"},"Setup Notes"),Object(a.b)("p",null,"INPUT AND OUTPUT LOGSTASH FILTERS\nStandard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs (01-inputs.conf and 99-outputs.conf) may be modified for your use.\nTo use the provided 01-inputs.conf and 99-outputs.conf versions, fill in the IP of the final rabbit host in 99-outputs.conf, and put the rabbitmq usernames and passwords into the logstash keystore.\nYour 01 and 99 conf files should not be overwritten by upgrades."),Object(a.b)("p",null,"To set up the keystore:  (note that logstash-keystore takes a minute to come back with a prompt)\nBe sure /usr/share/logstash/config exists\n(the full path, in case you need it: /usr/share/logstash/bin/logstash-keystore)\nCreate logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create\n    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)\n  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)\n```sh\nTo list the keys:\n")),Object(a.b)("p",null,"  $ sudo -E logstash-keystore list"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"To remove a key-value pair:\n\n```sh\n  $ sudo -E logstash-keystore remove <key name>\n")),Object(a.b)("p",null,'FLOW STITCHING - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use "-w 1".'),Object(a.b)("p",null,"See the comments in 04-stitching.conf to learn more about how complete flows are defined."))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var o=n(0),i=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,o,i=function(e,t){if(null==e)return{};var n,o,i={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var c=i.a.createContext({}),p=function(e){var t=i.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return i.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},h=i.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,b=u["".concat(s,".").concat(h)]||u[h]||d[h]||a;return n?i.a.createElement(b,r(r({ref:t},c),{},{components:n})):i.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var r={};for(var l in t)hasOwnProperty.call(t,l)&&(r[l]=t[l]);r.originalType=e,r.mdxType="string"==typeof e?e:o,s[1]=r;for(var c=2;c<a;c++)s[c]=n[c];return i.a.createElement.apply(null,s)}return i.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e2c161d9.57d50314.js b/assets/js/e2c161d9.00dc4bde.js
similarity index 95%
rename from assets/js/e2c161d9.57d50314.js
rename to assets/js/e2c161d9.00dc4bde.js
index e072d3d8..06e5c79f 100644
--- a/assets/js/e2c161d9.57d50314.js
+++ b/assets/js/e2c161d9.00dc4bde.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[143],{214:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},c={unversionedId:"pipeline/tstat",id:"pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/docs/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/next/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/tstat.md",version:"current",sidebar_label:"Tstat Data",sidebar:"Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/next/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump"}},s=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:s};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,c(c({ref:t},p),{},{components:n})):a.a.createElement(f,c({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,o[1]=c;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[138],{209:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},c={unversionedId:"pipeline/tstat",id:"pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/docs/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/next/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/tstat.md",version:"current",sidebar_label:"Tstat Data",sidebar:"Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/next/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/next/pipeline/nfdump"}},s=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:s};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,c(c({ref:t},p),{},{components:n})):a.a.createElement(f,c({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,o[1]=c;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e30bbbcf.e88d084c.js b/assets/js/e30bbbcf.b4e3431e.js
similarity index 96%
rename from assets/js/e30bbbcf.e88d084c.js
rename to assets/js/e30bbbcf.b4e3431e.js
index aee56240..9d51ea05 100644
--- a/assets/js/e30bbbcf.e88d084c.js
+++ b/assets/js/e30bbbcf.b4e3431e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[144],{215:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-1.2.11/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.11/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/nfdump.md",version:"1.2.11",sidebar_label:"Sflow/Netflow Data",sidebar:"version-1.2.11/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[139],{210:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-1.2.11/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.11/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/nfdump.md",version:"1.2.11",sidebar_label:"Sflow/Netflow Data",sidebar:"version-1.2.11/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e3649aa7.41406e34.js b/assets/js/e3649aa7.41406e34.js
deleted file mode 100644
index fd2f9062..00000000
--- a/assets/js/e3649aa7.41406e34.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[145],{216:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return l})),n.d(t,"default",(function(){return d}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"docker_upgrade",title:"Upgrading",sidebar_label:"Docker - Upgrading"},c={unversionedId:"deploy/docker_upgrade",id:"version-2.0.0/deploy/docker_upgrade",isDocsHomePage:!1,title:"Upgrading",description:"To upgrade a previous installment of the Dockerized pipeline, perform the following steps.",source:"@site/versioned_docs/version-2.0.0/deploy/docker_upgrade.md",slug:"/deploy/docker_upgrade",permalink:"/netsage-pipeline/docs/deploy/docker_upgrade",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_upgrade.md",version:"2.0.0",sidebar_label:"Docker - Upgrading",sidebar:"version-2.0.0/Pipeline",previous:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/deploy/docker_install_advanced"},next:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/deploy/docker_troubleshoot"}},l=[{value:"Shut things down",id:"shut-things-down",children:[]},{value:"Update Source Code",id:"update-source-code",children:[]},{value:"Check/Update Customization Files",id:"checkupdate-customization-files",children:[]},{value:"Update Docker Containers",id:"update-docker-containers",children:[]},{value:"Restart all the Docker Containers",id:"restart-all-the-docker-containers",children:[]},{value:"Delete old images and containers",id:"delete-old-images-and-containers",children:[]}],s={toc:l};function d(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"To upgrade a previous installment of the Dockerized pipeline, perform the following steps."),Object(a.b)("h3",{id:"shut-things-down"},"Shut things down"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"cd {netsage-pipeline directory}\ndocker-compose down\n")),Object(a.b)("p",null,"This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down."),Object(a.b)("h3",{id:"update-source-code"},"Update Source Code"),Object(a.b)("p",null,"To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. "),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git reset --hard\ngit pull origin master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing."))),Object(a.b)("p",null,'Run these three commands to select the new release you want to run. In the first, replace "{tag}" by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.'),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"git checkout -b {tag} \ngit pull\n./scripts/docker_select_version.sh\n")),Object(a.b)("p",null,"The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  "),Object(a.b)("h3",{id:"checkupdate-customization-files"},"Check/Update Customization Files"),Object(a.b)("p",null,"Occasionally, something may change which will necessitate editing your override and/or env file."),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare the new ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override_example.yml")," file to your ",Object(a.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml"),". Be sure to check to see if the version of nfdump has changed. Look for lines like ",Object(a.b)("inlineCode",{parentName:"p"},"image: netsage/nfdump-collector:"),". Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Also, look for",Object(a.b)("inlineCode",{parentName:"p"},'version: "x.x"')," at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)"))),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"Compare your ",Object(a.b)("inlineCode",{parentName:"p"},".env")," file with the new ",Object(a.b)("inlineCode",{parentName:"p"},"env.example")," file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. ")),Object(a.b)("li",{parentName:"ul"},Object(a.b)("p",{parentName:"li"},"If you used the Docker Advanced guide to make a ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_override.xml")," file, compare it to ",Object(a.b)("inlineCode",{parentName:"p"},"netsage_shared.xml")," to see if there are any changes. This is unlikely."))),Object(a.b)("h3",{id:"update-docker-containers"},"Update Docker Containers"),Object(a.b)("p",null,"This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose pull\n")),Object(a.b)("h3",{id:"restart-all-the-docker-containers"},"Restart all the Docker Containers"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker-compose up -d\n")),Object(a.b)("p",null,"This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors."),Object(a.b)("h3",{id:"delete-old-images-and-containers"},"Delete old images and containers"),Object(a.b)("p",null,"To keep things tidy, delete any old images and containers that are not being used."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image prune -a\ndocker container prune\n")),Object(a.b)("p",null,"To check which images you have"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"docker image ls\n")))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),d=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},p=function(e){var t=d(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),p=d(n),b=r,m=p["".concat(i,".").concat(b)]||p[b]||u[b]||a;return n?o.a.createElement(m,c(c({ref:t},s),{},{components:n})):o.a.createElement(m,c({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=b;var c={};for(var l in t)hasOwnProperty.call(t,l)&&(c[l]=t[l]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e436ce66.ffd97d6e.js b/assets/js/e436ce66.54f16243.js
similarity index 94%
rename from assets/js/e436ce66.ffd97d6e.js
rename to assets/js/e436ce66.54f16243.js
index c89dc0c6..6e559210 100644
--- a/assets/js/e436ce66.ffd97d6e.js
+++ b/assets/js/e436ce66.54f16243.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[146],{217:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return a})),t.d(n,"metadata",(function(){return p})),t.d(n,"toc",(function(){return i})),t.d(n,"default",(function(){return s}));var r=t(3),o=t(7),c=(t(0),t(232)),a={},p={unversionedId:"components/docker_pipeline",id:"version-1.2.7/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (importers and logstash) using:",source:"@site/versioned_docs/version-1.2.7/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_pipeline.md",version:"1.2.7"},i=[],l={toc:i};function s(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(c.b)("wrapper",Object(r.a)({},l,t,{components:n,mdxType:"MDXLayout"}),Object(c.b)("p",null,"Start up the pipeline (importers and logstash) using:"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(c.b)("p",null,"You can check the logs for each of the container by running"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(c.b)("p",null,"Shut down the pipeline using:"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}s.isMDXComponent=!0},232:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return b}));var r=t(0),o=t.n(r);function c(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function p(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?a(Object(t),!0).forEach((function(n){c(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):a(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},c=Object.keys(e);for(r=0;r<c.length;r++)t=c[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(r=0;r<c.length;r++)t=c[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var l=o.a.createContext({}),s=function(e){var n=o.a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):p(p({},n),e)),t},u=function(e){var n=s(e.components);return o.a.createElement(l.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},m=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,c=e.originalType,a=e.parentName,l=i(e,["components","mdxType","originalType","parentName"]),u=s(t),m=r,b=u["".concat(a,".").concat(m)]||u[m]||d[m]||c;return t?o.a.createElement(b,p(p({ref:n},l),{},{components:t})):o.a.createElement(b,p({ref:n},l))}));function b(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var c=t.length,a=new Array(c);a[0]=m;var p={};for(var i in n)hasOwnProperty.call(n,i)&&(p[i]=n[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var l=2;l<c;l++)a[l]=t[l];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,t)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[140],{211:function(e,n,t){"use strict";t.r(n),t.d(n,"frontMatter",(function(){return a})),t.d(n,"metadata",(function(){return p})),t.d(n,"toc",(function(){return i})),t.d(n,"default",(function(){return s}));var r=t(3),o=t(7),c=(t(0),t(228)),a={},p={unversionedId:"components/docker_pipeline",id:"version-1.2.7/components/docker_pipeline",isDocsHomePage:!1,title:"docker_pipeline",description:"Start up the pipeline (importers and logstash) using:",source:"@site/versioned_docs/version-1.2.7/components/docker_pipeline.md",slug:"/components/docker_pipeline",permalink:"/netsage-pipeline/docs/1.2.7/components/docker_pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_pipeline.md",version:"1.2.7"},i=[],l={toc:i};function s(e){var n=e.components,t=Object(o.a)(e,["components"]);return Object(c.b)("wrapper",Object(r.a)({},l,t,{components:n,mdxType:"MDXLayout"}),Object(c.b)("p",null,"Start up the pipeline (importers and logstash) using:"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(c.b)("p",null,"You can check the logs for each of the container by running"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(c.b)("p",null,"Shut down the pipeline using:"),Object(c.b)("pre",null,Object(c.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n")))}s.isMDXComponent=!0},228:function(e,n,t){"use strict";t.d(n,"a",(function(){return u})),t.d(n,"b",(function(){return b}));var r=t(0),o=t.n(r);function c(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function p(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?a(Object(t),!0).forEach((function(n){c(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):a(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,o=function(e,n){if(null==e)return{};var t,r,o={},c=Object.keys(e);for(r=0;r<c.length;r++)t=c[r],n.indexOf(t)>=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(r=0;r<c.length;r++)t=c[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var l=o.a.createContext({}),s=function(e){var n=o.a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):p(p({},n),e)),t},u=function(e){var n=s(e.components);return o.a.createElement(l.Provider,{value:n},e.children)},d={inlineCode:"code",wrapper:function(e){var n=e.children;return o.a.createElement(o.a.Fragment,{},n)}},m=o.a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,c=e.originalType,a=e.parentName,l=i(e,["components","mdxType","originalType","parentName"]),u=s(t),m=r,b=u["".concat(a,".").concat(m)]||u[m]||d[m]||c;return t?o.a.createElement(b,p(p({ref:n},l),{},{components:t})):o.a.createElement(b,p({ref:n},l))}));function b(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var c=t.length,a=new Array(c);a[0]=m;var p={};for(var i in n)hasOwnProperty.call(n,i)&&(p[i]=n[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,a[1]=p;for(var l=2;l<c;l++)a[l]=t[l];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,t)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e4c61278.03e947b6.js b/assets/js/e4c61278.03e947b6.js
deleted file mode 100644
index ba0eb866..00000000
--- a/assets/js/e4c61278.03e947b6.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[147],{218:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),r=n(7),a=(n(0),n(232)),i={id:"docker_dev_tag",title:"How to Release a New Version of the Pipeline",sidebar_label:"Making Releases"},l={unversionedId:"devel/docker_dev_tag",id:"version-2.0.0/devel/docker_dev_tag",isDocsHomePage:!1,title:"How to Release a New Version of the Pipeline",description:"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images..",source:"@site/versioned_docs/version-2.0.0/devel/tag.md",slug:"/devel/docker_dev_tag",permalink:"/netsage-pipeline/docs/devel/docker_dev_tag",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/tag.md",version:"2.0.0",sidebar_label:"Making Releases",sidebar:"version-2.0.0/Pipeline",previous:{title:"Revising Documentation",permalink:"/netsage-pipeline/docs/devel/docusaurus"}},s=[{value:"Make an RPM Release",id:"make-an-rpm-release",children:[]},{value:"In Github, Create a Release Tag",id:"in-github-create-a-release-tag",children:[]},{value:"To Build and Push Images Manually",id:"to-build-and-push-images-manually",children:[]},{value:"Building With Automation",id:"building-with-automation",children:[]},{value:"Test Docker Images",id:"test-docker-images",children:[]},{value:"Make Versioned Docs",id:"make-versioned-docs",children:[]},{value:"To Make New Nfdump-Collector Images",id:"to-make-new-nfdump-collector-images",children:[{value:"New Version of Logstash",id:"new-version-of-logstash",children:[]}]}],c={toc:s};function d(e){var t=e.components,n=Object(r.a)(e,["components"]);return Object(a.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images.."),Object(a.b)("h2",{id:"make-an-rpm-release"},"Make an RPM Release"),Object(a.b)("p",null,"Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments."),Object(a.b)("h2",{id:"in-github-create-a-release-tag"},"In Github, Create a Release Tag"),Object(a.b)("p",null,"Create a new Tag or Release in Github, eg, v1.2.11.\nBe sure to copy info from the CHANGES file into the Release description."),Object(a.b)("h2",{id:"to-build-and-push-images-manually"},"To Build and Push Images Manually"),Object(a.b)("p",null,"Below is the procedure to build pipeline_importer and pipeline_logstash images manually."),Object(a.b)("p",null,"Install docker-compose if not done already. See the Docker Installation instructions."),Object(a.b)("p",null,"Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"git clone https://github.com/netsage-project/netsage-pipeline.git\ncd netsage-pipeline\ngit checkout -b v1.2.11\n./scripts/docker_select_version.sh 1.2.11\n")),Object(a.b)("p",null,"Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo systemctl start docker\n$ sudo docker-compose -f docker-compose.build.yml build\n$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)\n     or  $ docker push $image:$tag                        (will push a specific image version)\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.\nIf that doesn't work, try adding this to /etc/hosts: ",Object(a.b)("inlineCode",{parentName:"p"},"67.219.148.138  mirrorlist.centos.org"),", and/or try ",Object(a.b)("inlineCode",{parentName:"p"},"yum install net-tools bridge-utils"),", and/or restart network.service then docker. "),Object(a.b)("p",null,"The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level)."),Object(a.b)("p",null,'It might be a good idea to test the images before pushing them. See "Test Docker Images" below.'),Object(a.b)("h2",{id:"building-with-automation"},"Building With Automation"),Object(a.b)("p",null,"???"),Object(a.b)("h2",{id:"test-docker-images"},"Test Docker Images"),Object(a.b)("p",null,"See the Docker installation instructions for details... "),Object(a.b)("p",null,"In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. "),Object(a.b)("p",null,"Run docker_select_version.sh if you haven't already, then start it up ",Object(a.b)("inlineCode",{parentName:"p"},"$ sudo docker-compose up -d"),". If there are local images, they'll be used, otherwise they'll be pulled from Docker Hub."),Object(a.b)("p",null,"After about 30 minutes, you should see flows in elasticsearch."),Object(a.b)("h2",{id:"make-versioned-docs"},"Make Versioned Docs"),Object(a.b)("p",null,"A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the ",Object(a.b)("strong",{parentName:"p"},"Docusaurus guide"),". "),Object(a.b)("h2",{id:"to-make-new-nfdump-collector-images"},"To Make New Nfdump-Collector Images"),Object(a.b)("p",null,"If a new version of nfdump has been released that we need, new nfdump-collector images need to be made."),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ git clone https://github.com/netsage-project/docker-nfdump-collector.git\n$ cd docker-nfdump-collector\n$ sudo systemctl start docker\n")),Object(a.b)("p",null,"To use squash: create a file at\xa0/etc/docker/daemon.json and put into it"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},' "experimental": true  \n "debug: false"\n')),Object(a.b)("p",null,"To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector\n$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    \n")),Object(a.b)("p",null,"To push to Docker Hub and quit docker"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"$ sudo docker login\n     provide your DockerHub login credentials\n$ sudo docker push netsage/nfdump-collector:$VER\n$ sudo systemctl stop docker\n")),Object(a.b)("p",null,"To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"sflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n...\nnetflow-collector:\n    image: netsage/nfdump-collector:alpine-1.6.23\n")),Object(a.b)("p",null,"Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade."),Object(a.b)("h3",{id:"new-version-of-logstash"},"New Version of Logstash"),Object(a.b)("p",null,"If a new version of logstash has been released that we want everyone to use,\n???"))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var o=n(0),r=n.n(o);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,r=function(e,t){if(null==e)return{};var n,o,r={},a=Object.keys(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o<a.length;o++)n=a[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var c=r.a.createContext({}),d=function(e){var t=r.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=d(e.components);return r.a.createElement(c.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.a.createElement(r.a.Fragment,{},t)}},b=r.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=d(n),b=o,m=u["".concat(i,".").concat(b)]||u[b]||p[b]||a;return n?r.a.createElement(m,l(l({ref:t},c),{},{components:n})):r.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<a;c++)i[c]=n[c];return r.a.createElement.apply(null,i)}return r.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e5ce8f83.b18dcb47.js b/assets/js/e5ce8f83.b18dcb47.js
deleted file mode 100644
index d1211c01..00000000
--- a/assets/js/e5ce8f83.b18dcb47.js
+++ /dev/null
@@ -1 +0,0 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[148],{219:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/docs/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/next/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/bare_metal_install.md",version:"current",sidebar_label:"Manual Installation",sidebar:"Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/next/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/next/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e746408e.8663b558.js b/assets/js/e746408e.d8730765.js
similarity index 99%
rename from assets/js/e746408e.8663b558.js
rename to assets/js/e746408e.d8730765.js
index dc25b906..87e4f9b6 100644
--- a/assets/js/e746408e.8663b558.js
+++ b/assets/js/e746408e.d8730765.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[149],{220:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.10/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",version:"1.2.10",sidebar_label:"Docker Installation",sidebar:"version-1.2.10/Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor (flow exporter/router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used, so it doesn't run and just create empty files."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  Change these only if required."),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "RNDNet New York - London 1" and "RNDNet New York - London 2". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). (For NetSage, another logstash pipeline on a remote server moves flows from this final rabbit queue into Elasticsearch.)"),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you could start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),h=o,m=d["".concat(i,".").concat(h)]||d[h]||b[h]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[141],{212:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_simple",title:"Docker Installation Guide",sidebar_label:"Docker Installation"},l={unversionedId:"deploy/docker_install_simple",id:"version-1.2.10/deploy/docker_install_simple",isDocsHomePage:!1,title:"Docker Installation Guide",description:"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",slug:"/deploy/docker_install_simple",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",version:"1.2.10",sidebar_label:"Docker Installation",sidebar:"version-1.2.10/Pipeline",previous:{title:"Manual Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"},next:{title:"Docker Advanced Options Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"}},s=[{value:"1. Set up Data Sources",id:"1-set-up-data-sources",children:[]},{value:"2. Clone the Netsage Pipeline Project",id:"2-clone-the-netsage-pipeline-project",children:[]},{value:"3. Create Docker-compose.override.yml",id:"3-create-docker-composeoverrideyml",children:[]},{value:"4. Create Environment File",id:"4-create-environment-file",children:[]},{value:"5. Choose Pipeline Version",id:"5-choose-pipeline-version",children:[]},{value:"Running the Collectors",id:"running-the-collectors",children:[]},{value:"Running the Collectors and Pipeline",id:"running-the-collectors-and-pipeline",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."),Object(r.b)("p",null,"The Docker containers included in the installation are"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"rabbit    (the local RabbitMQ server)"),Object(r.b)("li",{parentName:"ul"},"sflow-collector   (receives sflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"netflow-collector   (receives netflow data and writes nfcapd files)"),Object(r.b)("li",{parentName:"ul"},"importer   (reads nfcapd files and puts flows into a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)"),Object(r.b)("li",{parentName:"ul"},"ofelia   (cron-like downloading of files used by the logstash pipeline)")),Object(r.b)("p",null,"The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors."),Object(r.b)("h3",{id:"1-set-up-data-sources"},"1. Set up Data Sources"),Object(r.b)("p",null,"The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed."),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"sflow "),Object(r.b)("li",{parentName:"ul"},"netflow"),Object(r.b)("li",{parentName:"ul"},"tstat")),Object(r.b)("p",null,"At least one of these must be set up on a sensor (flow exporter/router), to provide the incoming flow data.\nYou can do this step later, but it will helpful to have it working first. "),Object(r.b)("p",null,"Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course."),Object(r.b)("p",null,'Tstat data should be sent directly to the logstash input rabbit queue "netsage_deidentifier_raw" on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)'),Object(r.b)("h3",{id:"2-clone-the-netsage-pipeline-project"},"2. Clone the Netsage Pipeline Project"),Object(r.b)("p",null,"If you haven't already, install ",Object(r.b)("a",{parentName:"p",href:"https://www.docker.com"},"Docker")," and ",Object(r.b)("a",{parentName:"p",href:"https://docs.docker.com/compose/install/"},"Docker Compose")," and clone this project"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git clone https://github.com/netsage-project/netsage-pipeline.git\n")),Object(r.b)("p",null,"(If you are upgrading to a new release, see the Upgrade section below!)"),Object(r.b)("p",null,"Then checkout the right version of the code."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"git checkout {tag}\n")),Object(r.b)("p",null,'Replace "{tag}" with the release version you intend to use, e.g., "v1.2.8".  ("Master" is the development version and is not intended for general use!)\n',Object(r.b)("inlineCode",{parentName:"p"},"git status")," will confirm which branch you are on, e.g., master or v1.2.8."),Object(r.b)("h3",{id:"3-create-docker-composeoverrideyml"},"3. Create Docker-compose.override.yml"),Object(r.b)("p",null,"Information in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.yml")," file tells docker which containers (processes) to run and sets various parameters for them.\nSettings in the ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose.override.yml")," file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten."),Object(r.b)("p",null,"Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp docker-compose.override_example.yml docker-compose.override.yml\n")),Object(r.b)("p",null,"By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don't need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn't run and simply create empty nfcapd files."),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you only have one collector, you should remove or comment out the section for the collector that is not used, so it doesn't run and just create empty files."))),Object(r.b)("p",null,"This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/"),". Each collector is namespaced by its type so the sflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/sflow/")," and the netflow collector will write data to ",Object(r.b)("inlineCode",{parentName:"p"},"/data/input_data/netflow/"),".  Change these only if required."),Object(r.b)("p",null,"Other lines in this file you can ignore for now. "),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose"))),Object(r.b)("h3",{id:"4-create-environment-file"},"4. Create Environment File"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Please copy ",Object(r.b)("inlineCode",{parentName:"p"},"env.example")," to ",Object(r.b)("inlineCode",{parentName:"p"},".env"),"  "),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp env.example .env \n")),Object(r.b)("p",{parentName:"p"},"then edit the .env file to set the sensor names"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sflowSensorName=my sflow sensor name\nnetflowSensorName=my netflow sensor name\n")),Object(r.b)("p",{parentName:"p"},"Simply change the names to unique identifiers (with spaces or not, no quotes) and you're good to go. "),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the ",Object(r.b)("inlineCode",{parentName:"p"},"meta.sensor_id"),' field. Choose names that are meaningful and unique.\nFor example, your sensor names might be "RNDNet New York Sflow" and "RNDNet Boston Netflow" or "RNDNet New York - London 1" and "RNDNet New York - London 2". Whatever makes sense in your situation.'))),Object(r.b)("ul",{parentName:"p"},Object(r.b)("li",{parentName:"ul"},"If you don't set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. "),Object(r.b)("li",{parentName:"ul"},"If you have only one collector, remove or comment out the line for the one you are not using."),Object(r.b)("li",{parentName:"ul"},'If you have more than one of the same type of collector, see the "Docker Advanced" documentation.')),Object(r.b)("p",{parentName:"p"},"Other settings of note in this file include the following. You will not necessarily need to change these, but be aware."),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"rabbit_output_host"),": this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on ",Object(r.b)("inlineCode",{parentName:"p"},"rabbit"),", ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). (For NetSage, another logstash pipeline on a remote server moves flows from this final rabbit queue into Elasticsearch.)"),Object(r.b)("p",{parentName:"p"},"The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.\n(See comments in the ","*","-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Aggregation_maps_path"),": the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  "),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"Inactivity_timeout"),": If more than inactivity_timeout seconds have passed between the 'start' of a flow and the 'start'\nof the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds\non the clock, assume the flow has ended."),Object(r.b)("div",{parentName:"p",className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don't meet the cutoff)"))),Object(r.b)("p",{parentName:"p"},Object(r.b)("strong",{parentName:"p"},"max_flow_timeout"),": If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.")),Object(r.b)("h3",{id:"5-choose-pipeline-version"},"5. Choose Pipeline Version"),Object(r.b)("p",null,"Once you've created the docker-compose.override.xml file and finished adjusting it for any customizations, you're ready to select which version Docker should run."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"./scripts/docker_select_version.sh\n")),Object(r.b)("p",null,"When prompted, select the ",Object(r.b)("strong",{parentName:"p"},"same version")," you checked out earlier.\nThis script will replace the version numbers of docker images in the docker-compose files with the correct values."),Object(r.b)("h2",{id:"running-the-collectors"},"Running the Collectors"),Object(r.b)("p",null,"After selecting the version to run, you could start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. "),Object(r.b)("p",null,"(Or see the next section for how to start all the containers, including the collectors.)"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d sflow-collector netflow-collector\n")),Object(r.b)("p",null,"If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)"),Object(r.b)("h2",{id:"running-the-collectors-and-pipeline"},"Running the Collectors and Pipeline"),Object(r.b)("p",null,Object(r.b)("p",{parentName:"p"},"Start up the pipeline (all containers) using:"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d\n")),Object(r.b)("p",{parentName:"p"},'This will also restart any containers/processes that have died. "-d" runs containers in the background.'),Object(r.b)("p",{parentName:"p"},"You can see the status of the containers and whether any have died (exited) using"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose ps\n")),Object(r.b)("p",{parentName:"p"},"To check the logs for each of the containers, run"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose logs\n")),Object(r.b)("p",{parentName:"p"},"Add ",Object(r.b)("inlineCode",{parentName:"p"},"-f")," or, e.g., ",Object(r.b)("inlineCode",{parentName:"p"},"-f logstash")," to see new log messages as they arrive.  ",Object(r.b)("inlineCode",{parentName:"p"},"--timestamps"),", ",Object(r.b)("inlineCode",{parentName:"p"},"--tail"),",  and ",Object(r.b)("inlineCode",{parentName:"p"},"--since")," are also useful -- look up details in Docker documentation."),Object(r.b)("p",{parentName:"p"},"To shut down the pipeline (all containers) use"),Object(r.b)("pre",{parentName:"p"},Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose down\n"))))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),p=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=p(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),h=o,m=d["".concat(i,".").concat(h)]||d[h]||b[h]||r;return n?a.a.createElement(m,l(l({ref:t},c),{},{components:n})):a.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e7670743.b2234f2e.js b/assets/js/e7670743.fb9bbf42.js
similarity index 99%
rename from assets/js/e7670743.b2234f2e.js
rename to assets/js/e7670743.fb9bbf42.js
index 8c3662da..e8c21b0e 100644
--- a/assets/js/e7670743.b2234f2e.js
+++ b/assets/js/e7670743.fb9bbf42.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[150],{221:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(232)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.10/deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",version:"1.2.10",sidebar_label:"Manual Installation",sidebar:"version-1.2.10/Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[142],{213:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var a=n(3),o=n(7),i=(n(0),n(228)),l={id:"bare_metal_install",title:"Manual Installation Guide",sidebar_label:"Manual Installation"},r={unversionedId:"deploy/bare_metal_install",id:"version-1.2.10/deploy/bare_metal_install",isDocsHomePage:!1,title:"Manual Installation Guide",description:"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.",source:"@site/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",slug:"/deploy/bare_metal_install",permalink:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",version:"1.2.10",sidebar_label:"Manual Installation",sidebar:"version-1.2.10/Pipeline",previous:{title:"Choosing an Installation Procedure",permalink:"/netsage-pipeline/docs/1.2.10/deploy/choose_install"},next:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"}},s=[{value:"Data sources",id:"data-sources",children:[]},{value:"Installing the Prerequisites",id:"installing-the-prerequisites",children:[{value:"Installing nfdump",id:"installing-nfdump",children:[]},{value:"Installing RabbitMQ",id:"installing-rabbitmq",children:[]},{value:"Installing Logstash",id:"installing-logstash",children:[]},{value:"Installing the EPEL repo",id:"installing-the-epel-repo",children:[]},{value:"Installing the GlobalNOC Open Source repo",id:"installing-the-globalnoc-open-source-repo",children:[]}]},{value:"Installing the Pipeline (Importer and Logstash configs)",id:"installing-the-pipeline-importer-and-logstash-configs",children:[]},{value:"Importer Configuration",id:"importer-configuration",children:[{value:"Setting up the shared config file",id:"setting-up-the-shared-config-file",children:[]},{value:"Setting up the Importer config file",id:"setting-up-the-importer-config-file",children:[]}]},{value:"Logstash Setup Notes",id:"logstash-setup-notes",children:[]},{value:"Start Logstash",id:"start-logstash",children:[]},{value:"Start the Importer",id:"start-the-importer",children:[]},{value:"Cron jobs",id:"cron-jobs",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(a.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("p",null,"This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."),Object(i.b)("h2",{id:"data-sources"},"Data sources"),Object(i.b)("p",null,"The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed."),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"sflow or netflow"),Object(i.b)("li",{parentName:"ol"},"tstat")),Object(i.b)("p",null,"At least one of these must be set up on a sensor to provide the incoming flow data."),Object(i.b)("p",null,"Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it."),Object(i.b)("p",null,"Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data."),Object(i.b)("h2",{id:"installing-the-prerequisites"},"Installing the Prerequisites"),Object(i.b)("h3",{id:"installing-nfdump"},"Installing nfdump"),Object(i.b)("p",null,"The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.\nThe Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. "),Object(i.b)("p",null,"Nfdump is ",Object(i.b)("em",{parentName:"p"},"not")," listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, ",Object(i.b)("inlineCode",{parentName:"p"},"yum install nfdump")," should work.\nFlow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. "),Object(i.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(i.b)("div",{parentName:"div",className:"admonition-heading"},Object(i.b)("h5",{parentName:"div"},Object(i.b)("span",{parentName:"h5",className:"admonition-icon"},Object(i.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(i.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(i.b)("div",{parentName:"div",className:"admonition-content"},Object(i.b)("p",{parentName:"div"},"It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included."))),Object(i.b)("p",null,"If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. "),Object(i.b)("h3",{id:"installing-rabbitmq"},"Installing RabbitMQ"),Object(i.b)("p",null,"The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package)."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# yum install rabbitmq-server\n\n")),Object(i.b)("p",null,"Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service rabbitmq-server start \n          or # systemctl start rabbitmq-server.service\n")),Object(i.b)("h3",{id:"installing-logstash"},"Installing Logstash"),Object(i.b)("p",null,"See the logstash documentation. We are currently using Version 7.10."),Object(i.b)("h3",{id:"installing-the-epel-repo"},"Installing the EPEL repo"),Object(i.b)("p",null,"Some of our dependencies come from the EPEL repo. To install this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install epel-release\n")),Object(i.b)("h3",{id:"installing-the-globalnoc-open-source-repo"},"Installing the GlobalNOC Open Source repo"),Object(i.b)("p",null,"The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo."),Object(i.b)("p",null,"For Red Hat/CentOS 6, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc6.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc6]\nname=GlobalNOC Public el6 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6\n")),Object(i.b)("p",null,"For Red Hat/CentOS 7, create ",Object(i.b)("inlineCode",{parentName:"p"},"/etc/yum.repos.d/grnoc7.repo")," with the following content."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[grnoc7]\nname=GlobalNOC Public el7 Packages - $basearch\nbaseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch\nenabled=1\ngpgcheck=1\ngpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7\n")),Object(i.b)("p",null,"The first time you install packages from the repo, you will have to accept the GlobalNOC repo key."),Object(i.b)("h2",{id:"installing-the-pipeline-importer-and-logstash-configs"},"Installing the Pipeline (Importer and Logstash configs)"),Object(i.b)("p",null,"Install it like this:"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"[root@host ~]# yum install grnoc-netsage-deidentifier\n")),Object(i.b)("p",null,"Pipeline components:"),Object(i.b)("ol",null,Object(i.b)("li",{parentName:"ol"},"Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise."),Object(i.b)("li",{parentName:"ol"},"Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)"),Object(i.b)("li",{parentName:"ol"},"Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  "),Object(i.b)("li",{parentName:"ol"},"Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.")),Object(i.b)("p",null,"Nothing will automatically start after installation as we need to move on to configuration. "),Object(i.b)("h2",{id:"importer-configuration"},"Importer Configuration"),Object(i.b)("p",null,"Configuration files of interest are"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging.conf - logging config"),Object(i.b)("li",{parentName:"ul"},"/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled")),Object(i.b)("h3",{id:"setting-up-the-shared-config-file"},"Setting up the shared config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_shared.xml")),Object(i.b)("p",null,"There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file."),Object(i.b)("p",null,"The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},'<collection>\n     \x3c!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --\x3e\n         <flow-path>/path/to/netflow-files/</flow-path>\n\n     \x3c!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --\x3e\n         <sensor>Netflow Sensor 1</sensor>\n\n     \x3c!-- Flow type - sflow or netflow (defaults to netflow) --\x3e\n         <flow-type>sflow</flow-type>\n\n     \x3c!-- "instance" goes along with sensor.  This is to identify various instances if a sensor has --\x3e\n     \x3c!-- more than one "stream" / data collection.  Defaults to 0. --\x3e\n     \x3c!-- <instance>1</instance> --\x3e\n\n     \x3c!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --\x3e\n     \x3c!-- <router-address></router-address> --\x3e\n</collection>\n')),Object(i.b)("p",null,"Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. "),Object(i.b)("p",null,"There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) "),Object(i.b)("p",null,"Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL."),Object(i.b)("p",null,"If you're running a default RabbitMQ config, which is open only to 'localhost' as guest/guest, you won't need to change anything here."),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre"},"  \x3c!-- rabbitmq connection info --\x3e\n  <rabbit_input>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_input>\n\n  <rabbit_output>\n    <host>127.0.0.1</host>\n    <port>5672</port>\n    <username>guest</username>\n    <password>guest</password>\n    <ssl>0</ssl>\n    <batch_size>100</batch_size>\n    <vhost>/</vhost>\n    <durable>1</durable> \x3c!-- Whether the rabbit queue is 'durable' (don't change this unless you have a reason) --\x3e\n  </rabbit_output>\n")),Object(i.b)("h3",{id:"setting-up-the-importer-config-file"},"Setting up the Importer config file"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml")),Object(i.b)("p",null,"This file has a few more setting specific to the Importer component which you may like to adjust.  "),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},"Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  "),Object(i.b)("li",{parentName:"ul"},'(The Importer does not actually use an input rabbit queue, so we add a "fake" one here.)'),Object(i.b)("li",{parentName:"ul"},"Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded."),Object(i.b)("li",{parentName:"ul"},"Min-file-age is used to be sure files are complete before being read. "),Object(i.b)("li",{parentName:"ul"},"Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. "),Object(i.b)("li",{parentName:"ul"},"Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file."),Object(i.b)("li",{parentName:"ul"},"Keep num-processes set to 1.")),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-xml"},'<config>\n  \x3c!--  NOTE: Values here override those in the shared config --\x3e\n\n  \x3c!-- rabbitmq queues --\x3e\n  <rabbit_input>\n    <queue>netsage_deidentifier_netflow_fake</queue>\n    <channel>2</channel>\n  </rabbit_input>\n\n  <rabbit_output>\n    <channel>3</channel>\n    <queue>netsage_deidentifier_raw</queue>\n  </rabbit_output>\n\n  <worker>\n    \x3c!-- How many flows to process at once --\x3e\n        <flow-batch-size>100</flow-batch-size>\n\n    \x3c!-- How many concurrent workers should perform the necessary operations --\x3e\n        <num-processes>1</num-processes>\n\n    \x3c!-- path to nfdump executable (defaults to /usr/bin/nfdump) --\x3e\n    \x3c!--   <nfdump-path>/path/to/nfdump</nfdump-path>  --\x3e\n\n    \x3c!-- Where to store the cache, where it tracks what files it has/hasn\'t read --\x3e\n        <cache-file>/var/cache/netsage/netflow_importer.cache</cache-file>\n\n    \x3c!-- The minium flow size threshold - will not  import any flows smaller than this --\x3e\n    \x3c!-- Defaults to 500M  --\x3e\n        <min-bytes>100000000</min-bytes> \n\n    \x3c!-- Do not import nfcapd files younger than min-file-age\n        The value must match /^(\\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are\n        "day(s)", "week(s)", "month(s)", "year(s)", "hour(s)", "minute(s)" and "second(s)", respectively"\n        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm\n        Default: 0 (no minimum age) \n    --\x3e\n        <min-file-age>10m</min-file-age> \n\n    \x3c!-- cull-enable: whether to cull processed flow data files --\x3e\n    \x3c!-- default: no culling; set to 1 to turn culling on --\x3e\n    \x3c!--    <cull-enable>1</cull-enable>  --\x3e\n\n    \x3c!-- cull-tty: cull time to live, in days --\x3e\n    \x3c!-- number of days to retain imported data files before deleting them; default: 3 --\x3e\n    \x3c!--    <cull-ttl>5</cull-ttl>  --\x3e\n  </worker>\n\n  <master>\n    \x3c!-- where should we write the daemon pid file to --\x3e\n        <pid-file>/var/run/netsage-netflow-importer-daemon.pid</pid-file>\n  </master>\n\n</config>\n')),Object(i.b)("h2",{id:"logstash-setup-notes"},"Logstash Setup Notes"),Object(i.b)("p",null,"Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use."),Object(i.b)("p",null,"The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. "),Object(i.b)("p",null,"When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs."),Object(i.b)("p",null,'FOR FLOW STITCHING/AGGREGATION - IMPORTANT!\nFlow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!\nBe sure to set "pipeline.workers: 1" in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use "-w 1".'),Object(i.b)("h2",{id:"start-logstash"},"Start Logstash"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service logstash start \n          or # systemctl start logstash.service\n")),Object(i.b)("p",null,"It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log."),Object(i.b)("p",null,'When logstash is stopped, any flows currently "in the aggregator" will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. '),Object(i.b)("h2",{id:"start-the-importer"},"Start the Importer"),Object(i.b)("p",null,"Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--config [file]")," - specify which config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--sharedconfig [file]")," - specify which shared config file to read"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--logging [file]")," - the logging config"),Object(i.b)("p",null,Object(i.b)("inlineCode",{parentName:"p"},"--nofork")," - run in foreground (do not daemonize)"),Object(i.b)("pre",null,Object(i.b)("code",{parentName:"pre",className:"language-sh"},"[root@host ~]# /sbin/service netsage-netflow-importer start \n          or # systemctl start netsage-netflow-importer.service\n")),Object(i.b)("p",null,"The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. "),Object(i.b)("h2",{id:"cron-jobs"},"Cron jobs"),Object(i.b)("p",null,"Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. "))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var a=n(0),o=n.n(a);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,o=function(e,t){if(null==e)return{};var n,a,o={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},h=o.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),h=a,b=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?o.a.createElement(b,r(r({ref:t},c),{},{components:n})):o.a.createElement(b,r({ref:t},c))}));function b(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,l=new Array(i);l[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r.mdxType="string"==typeof e?e:a,l[1]=r;for(var c=2;c<i;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/e987ec87.a4179a0d.js b/assets/js/e987ec87.f66b2f78.js
similarity index 96%
rename from assets/js/e987ec87.a4179a0d.js
rename to assets/js/e987ec87.f66b2f78.js
index 5097244e..ac503a11 100644
--- a/assets/js/e987ec87.a4179a0d.js
+++ b/assets/js/e987ec87.f66b2f78.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[151],{222:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.11/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.11/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_first_steps.md",version:"1.2.11"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[143],{214:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"version-1.2.11/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-1.2.11/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/1.2.11/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_first_steps.md",version:"1.2.11"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ebfaece6.fbef59fb.js b/assets/js/ebfaece6.d97235a2.js
similarity index 96%
rename from assets/js/ebfaece6.fbef59fb.js
rename to assets/js/ebfaece6.d97235a2.js
index d5dda875..22795e1b 100644
--- a/assets/js/ebfaece6.fbef59fb.js
+++ b/assets/js/ebfaece6.d97235a2.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[152],{223:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(232)),a={id:"pipeline",title:"Pipeline",sidebar_label:"Intro"},l={unversionedId:"pipeline",id:"version-1.2.6/pipeline",isDocsHomePage:!1,title:"Pipeline",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.6/pipeline.md",slug:"/pipeline",permalink:"/netsage-pipeline/docs/1.2.6/pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline.md",version:"1.2.6",sidebar_label:"Intro",sidebar:"version-1.2.6/Pipeline",next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer"}},c=[{value:"Components",id:"components",children:[]},{value:"Sensors and Data Collection",id:"sensors-and-data-collection",children:[{value:"Importer",id:"importer",children:[]}]}],p={toc:c};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h1",{id:"description"},"Description"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc."),Object(o.b)("h2",{id:"components"},"Components"),Object(o.b)("p",null,"The Pipeline is made of the following components (Currently)"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"Importer"),"  (Collection of perl scripts)",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_importer"},"doc")))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Elastic Logstash")," Performs a variety of transformation on the data",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_logstash"},"doc")," "))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ")," used for message passing and queing of tasks.")),Object(o.b)("h2",{id:"sensors-and-data-collection"},"Sensors and Data Collection"),Object(o.b)("p",null,'"Testpoints" or "sensors" collect flow data (',Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). '),Object(o.b)("p",null,"Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files."),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return f}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),s=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},b=function(e){var t=s(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),b=s(n),d=r,f=b["".concat(a,".").concat(d)]||b[d]||u[d]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var p=2;p<o;p++)a[p]=n[p];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[144],{215:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return s}));var r=n(3),i=n(7),o=(n(0),n(228)),a={id:"pipeline",title:"Pipeline",sidebar_label:"Intro"},l={unversionedId:"pipeline",id:"version-1.2.6/pipeline",isDocsHomePage:!1,title:"Pipeline",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-1.2.6/pipeline.md",slug:"/pipeline",permalink:"/netsage-pipeline/docs/1.2.6/pipeline",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline.md",version:"1.2.6",sidebar_label:"Intro",sidebar:"version-1.2.6/Pipeline",next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.6/pipeline_importer"}},c=[{value:"Components",id:"components",children:[]},{value:"Sensors and Data Collection",id:"sensors-and-data-collection",children:[{value:"Importer",id:"importer",children:[]}]}],p={toc:c};function s(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h1",{id:"description"},"Description"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc."),Object(o.b)("h2",{id:"components"},"Components"),Object(o.b)("p",null,"The Pipeline is made of the following components (Currently)"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"Importer"),"  (Collection of perl scripts)",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_importer"},"doc")))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Elastic Logstash")," Performs a variety of transformation on the data",Object(o.b)("ul",{parentName:"li"},Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"pipeline_logstash"},"doc")," "))),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ")," used for message passing and queing of tasks.")),Object(o.b)("h2",{id:"sensors-and-data-collection"},"Sensors and Data Collection"),Object(o.b)("p",null,'"Testpoints" or "sensors" collect flow data (',Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). '),Object(o.b)("p",null,"Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files."),Object(o.b)("h3",{id:"importer"},"Importer"),Object(o.b)("p",null,"A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue."))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return b})),n.d(t,"b",(function(){return f}));var r=n(0),i=n.n(r);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,i=function(e,t){if(null==e)return{};var n,r,i={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),s=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},b=function(e){var t=s(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},d=i.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,a=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),b=s(n),d=r,f=b["".concat(a,".").concat(d)]||b[d]||u[d]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,a=new Array(o);a[0]=d;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var p=2;p<o;p++)a[p]=n[p];return i.a.createElement.apply(null,a)}return i.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f0b0282f.243b745a.js b/assets/js/f0b0282f.243b745a.js
new file mode 100644
index 00000000..6bd2a16d
--- /dev/null
+++ b/assets/js/f0b0282f.243b745a.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[145],{216:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return s})),n.d(t,"metadata",(function(){return r})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return d}));var i=n(3),a=n(7),o=(n(0),n(228)),s={id:"logstash",title:"Logstash Pipeline",sidebar_label:"Logstash"},r={unversionedId:"pipeline/logstash",id:"version-2.1.0/pipeline/logstash",isDocsHomePage:!1,title:"Logstash Pipeline",description:"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance.",source:"@site/versioned_docs/version-2.1.0/pipeline/logstash.md",slug:"/pipeline/logstash",permalink:"/netsage-pipeline/docs/pipeline/logstash",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/logstash.md",version:"2.1.0",sidebar_label:"Logstash",sidebar:"version-2.1.0/Pipeline",previous:{title:"Importer",permalink:"/netsage-pipeline/docs/pipeline/importer"},next:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/pipeline/elastic"}},c=[{value:"Logstash Sequence",id:"logstash-sequence",children:[{value:"01-input-rabbit.conf",id:"01-input-rabbitconf",children:[]},{value:"10-preliminaries.conf",id:"10-preliminariesconf",children:[]},{value:"15-sensor-specific-changes.conf",id:"15-sensor-specific-changesconf",children:[]},{value:"20-add_id.conf",id:"20-add_idconf",children:[]},{value:"40-aggregation.conf",id:"40-aggregationconf",children:[]},{value:"45-geoip-tagging.conf",id:"45-geoip-taggingconf",children:[]},{value:"50-asn.conf",id:"50-asnconf",children:[]},{value:"53-caida-org.conf",id:"53-caida-orgconf",children:[]},{value:"55-member-orgs.conf",id:"55-member-orgsconf",children:[]},{value:"60-scireg-tagging-fakegeoip.conf",id:"60-scireg-tagging-fakegeoipconf",children:[]},{value:"70-deidentify.conf",id:"70-deidentifyconf",children:[]},{value:"80-privatize.org.conf",id:"80-privatizeorgconf",children:[]},{value:"88-preferred-location-org.conf",id:"88-preferred-location-orgconf",children:[]},{value:"90-additional-fields.conf",id:"90-additional-fieldsconf",children:[]},{value:"95-cleanup.conf",id:"95-cleanupconf",children:[]},{value:"98-post-process.conf",id:"98-post-processconf",children:[]},{value:"99-output-rabbit.conf",id:"99-output-rabbitconf",children:[]},{value:"Final Stage",id:"final-stage",children:[]}]},{value:"Field names",id:"field-names",children:[]}],l={toc:c};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(o.b)("wrapper",Object(i.a)({},l,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("p",null,"The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. "),Object(o.b)("p",null,'Logstash config files invoke various logstash "filters" and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.'),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"All ","*",".conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming 'path.config: \"/etc/logstash/conf.d/*.conf\"' in /etc/logstash/pipelines.yml)."),Object(o.b)("li",{parentName:"ul"},"If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs."),Object(o.b)("li",{parentName:"ul"},"MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) ",Object(o.b)("strong",{parentName:"li"},"NOTE that new versions won't be used in the pipeline until logstash is restarted.")," There is a cron file to do this also, though it's not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf."),Object(o.b)("li",{parentName:"ul"},"Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.")),Object(o.b)("h2",{id:"logstash-sequence"},"Logstash Sequence"),Object(o.b)("p",null,"The main things done in each conf file are as follows."),Object(o.b)("h3",{id:"01-input-rabbitconf"},"01-input-rabbit.conf"),Object(o.b)("p",null,'Reads flows from a rabbitmq queue. (The ".disabled" extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)'),Object(o.b)("h3",{id:"10-preliminariesconf"},"10-preliminaries.conf"),Object(o.b)("p",null,"Drops flows to or from private IP addresses;\nconverts any timestamps in milliseconds to seconds;\ndrops events with timestamps more than a year in the past or (10 sec) in the future;\nsets duration and rates to 0 if duration is <= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)"),Object(o.b)("h3",{id:"15-sensor-specific-changesconf"},"15-sensor-specific-changes.conf"),Object(o.b)("p",null,"Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False."),Object(o.b)("h3",{id:"20-add_idconf"},"20-add_id.conf"),Object(o.b)("p",null,"Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. "),Object(o.b)("h3",{id:"40-aggregationconf"},"40-aggregation.conf"),Object(o.b)("p",null,"Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together."),Object(o.b)("li",{parentName:"ul"},"If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes."),Object(o.b)("li",{parentName:"ul"},'There is another "timeout" setting which is basically the maximum duration of a stitched flow (default: 24 hr).'),Object(o.b)("li",{parentName:"ul"},'When logstash shuts down, any flows "in the aggregator" will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. '),Object(o.b)("li",{parentName:"ul"},"Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file."),Object(o.b)("li",{parentName:"ul"},"Tstat flows come in already complete, so no aggregation is done on those flows.")),Object(o.b)("h3",{id:"45-geoip-taggingconf"},"45-geoip-tagging.conf"),Object(o.b)("p",null,'Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;\nif the destination IP is in the multicast range, sets the destination Organization, Country, and Continent to "Multicast".'),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses GeoLite2 data created by MaxMind, available from ",Object(o.b)("a",{parentName:"em",href:"http://www.maxmind.com"},"www.maxmind.com"),".")),Object(o.b)("h3",{id:"50-asnconf"},"50-asn.conf"),Object(o.b)("p",null,"Normally with sflow and netflow, flows come in with source and destination ASNs.  If there is no ASN in the input event; or the input ASN is 0, 4294967295, or 23456, or it is a private ASN, tries to get an ASN by IP from the MaxMind ASN database.\nSets ASN to -1 if it is unavailable for any reason."),Object(o.b)("h3",{id:"53-caida-orgconf"},"53-caida-org.conf"),Object(o.b)("p",null,"Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file."),Object(o.b)("p",null,Object(o.b)("em",{parentName:"p"},"This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see ",Object(o.b)("a",{parentName:"em",href:"http://www.caida.org/data/as-organizations"},"www.caida.org"),".")," "),Object(o.b)("h3",{id:"55-member-orgsconf"},"55-member-orgs.conf"),Object(o.b)("p",null,"Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.\nThis allows entities which don't own their own ASs to be listed as the src or dst Organization."),Object(o.b)("p",null,"Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job."),Object(o.b)("h3",{id:"60-scireg-tagging-fakegeoipconf"},"60-scireg-tagging-fakegeoip.conf"),Object(o.b)("p",null,"Uses a fake geoip database containing ",Object(o.b)("a",{parentName:"p",href:"http://scienceregistry.grnoc.iu.edu"},"Science Registry")," information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;\nremoves Registry fields we don't need to save to elasticsearch."),Object(o.b)("p",null,"Notes: "),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"The ",Object(o.b)("a",{parentName:"li",href:"https://scienceregistry.netsage.global/rdb/"},"Science Registry"),' stores human-curated information about various "resources". Resources are sources and destinations of flows.'),Object(o.b)("li",{parentName:"ul"},'The Science Registry "fake geoip database" is updated weekly and can be downloaded via wget in a cron job (provided in the installation).')),Object(o.b)("h3",{id:"70-deidentifyconf"},"70-deidentify.conf"),Object(o.b)("p",null,"Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x's in order to deidentify them."),Object(o.b)("h3",{id:"80-privatizeorgconf"},"80-privatize.org.conf"),Object(o.b)("p",null,'Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).\nIf the ASN is one of those listed, completely replaces the IP with x\'s, sets the location to central Autralia, sets all organizations to "AARNet", removes all Projects.'),Object(o.b)("h3",{id:"88-preferred-location-orgconf"},"88-preferred-location-org.conf"),Object(o.b)("p",null,"Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields."),Object(o.b)("h3",{id:"90-additional-fieldsconf"},"90-additional-fields.conf"),Object(o.b)("p",null,"Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage's use):"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)"),Object(o.b)("li",{parentName:"ul"},"country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)"),Object(o.b)("li",{parentName:"ul"},"is_network_testing = yes, no  (yes if discipline from the science registry is 'CS.Network Testing and Monitoring' or port = 5001, 5101, or 5201)"),Object(o.b)("li",{parentName:"ul"},"es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)")),Object(o.b)("h3",{id:"95-cleanupconf"},"95-cleanup.conf"),Object(o.b)("p",null,"Does small misc. tasks at the end like rename, remove, or convert fields"),Object(o.b)("h3",{id:"98-post-processconf"},"98-post-process.conf"),Object(o.b)("p",null,"Adds @exit_time and @processing_time (these are mainly for developers)"),Object(o.b)("h3",{id:"99-output-rabbitconf"},"99-output-rabbit.conf"),Object(o.b)("p",null,'Sends results to a final RabbitMQ queue. (".disabled" can be removed from other output configs to send flows to other places)'),Object(o.b)("h3",{id:"final-stage"},"Final Stage"),Object(o.b)("p",null,"In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. "),Object(o.b)("h2",{id:"field-names"},"Field names"),Object(o.b)("p",null,"The fields used/created in Logstash (and saved to Elasticsearch) are listed in the ",Object(o.b)("a",{parentName:"p",href:"elastic"},"Elasticsearch doc"),"."))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return f})),n.d(t,"b",(function(){return h}));var i=n(0),a=n.n(i);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,i,a=function(e,t){if(null==e)return{};var n,i,a={},o=Object.keys(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(i=0;i<o.length;i++)n=o[i],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=a.a.createContext({}),d=function(e){var t=a.a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},f=function(e){var t=d(e.components);return a.a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},p=a.a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),f=d(n),p=i,h=f["".concat(s,".").concat(p)]||f[p]||u[p]||o;return n?a.a.createElement(h,r(r({ref:t},l),{},{components:n})):a.a.createElement(h,r({ref:t},l))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=p;var r={};for(var c in t)hasOwnProperty.call(t,c)&&(r[c]=t[c]);r.originalType=e,r.mdxType="string"==typeof e?e:i,s[1]=r;for(var l=2;l<o;l++)s[l]=n[l];return a.a.createElement.apply(null,s)}return a.a.createElement.apply(null,n)}p.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f1ddb0ed.0d7a9793.js b/assets/js/f1ddb0ed.4cbf6601.js
similarity index 99%
rename from assets/js/f1ddb0ed.0d7a9793.js
rename to assets/js/f1ddb0ed.4cbf6601.js
index 9e476b8f..e653c54b 100644
--- a/assets/js/f1ddb0ed.0d7a9793.js
+++ b/assets/js/f1ddb0ed.4cbf6601.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[153],{224:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(232)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.10/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",version:"1.2.10",sidebar_label:"Docker Advanced Options",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"a. Edit docker-compose.override.yml",id:"a-edit-docker-composeoverrideyml",children:[]},{value:"b.  Edit netsage_override.xml",id:"b--edit-netsage_overridexml",children:[]},{value:"c. Edit environment file",id:"c-edit-environment-file",children:[]},{value:"d. Running the new collector",id:"d-running-the-new-collector",children:[]}]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"For Data Saved to an NFS Volume",id:"for-data-saved-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but you only want to keep some of them (eg, only send some of them to NetSage), use this option. The collectors and importer will process all flows, but in the logstash pipeline, those that do not have src_ifindex or dst_inindex equal to one of the listed interfaces will be dropped. "),Object(r.b)("p",null,'In the .env file, uncomment the apprpriate section and enter the information required. Be sure "True" is capitalized as shown and list all the ifindex values of flows that should be kept and passed on to NetSage. You may enter one or more ifindex values. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_flag=True\nifindex_filter_keep=123,456\n")),Object(r.b)("p",null,"In this case, only flows that have src_ifindex = 123 or src_ifindex = 456 or dst_ifindex = 123 or dst_ifindex = 456 will be kept. All others will be dropped."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that come through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id in ElasticSearch) changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use."),Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",null,"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))))}d.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return u}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),h=o,u=p["".concat(i,".").concat(h)]||p[h]||m[h]||r;return n?a.a.createElement(u,l(l({ref:t},c),{},{components:n})):a.a.createElement(u,l({ref:t},c))}));function u(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[146],{217:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return d}));var o=n(3),a=n(7),r=(n(0),n(228)),i={id:"docker_install_advanced",title:"Docker Advanced Options Guide",sidebar_label:"Docker Advanced Options"},l={unversionedId:"deploy/docker_install_advanced",id:"version-1.2.10/deploy/docker_install_advanced",isDocsHomePage:!1,title:"Docker Advanced Options Guide",description:"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.",source:"@site/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",slug:"/deploy/docker_install_advanced",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",version:"1.2.10",sidebar_label:"Docker Advanced Options",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Installation Guide",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"},next:{title:"Upgrading",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"}},s=[{value:"To Add an Additional Sflow or Netflow Collector",id:"to-add-an-additional-sflow-or-netflow-collector",children:[{value:"a. Edit docker-compose.override.yml",id:"a-edit-docker-composeoverrideyml",children:[]},{value:"b.  Edit netsage_override.xml",id:"b--edit-netsage_overridexml",children:[]},{value:"c. Edit environment file",id:"c-edit-environment-file",children:[]},{value:"d. Running the new collector",id:"d-running-the-new-collector",children:[]}]},{value:"To Keep Only Flows From Certain Interfaces",id:"to-keep-only-flows-from-certain-interfaces",children:[]},{value:"To Change a Sensor Name Depending on the Interface Used",id:"to-change-a-sensor-name-depending-on-the-interface-used",children:[]},{value:"To Do Sampling Rate Corrections in Logstash",id:"to-do-sampling-rate-corrections-in-logstash",children:[]},{value:"To Change How Long Nfcapd Files Are Kept",id:"to-change-how-long-nfcapd-files-are-kept",children:[]},{value:"To Customize Java Settings / Increase Memory Available for Lostash",id:"to-customize-java-settings--increase-memory-available-for-lostash",children:[]},{value:"To Bring up Kibana and Elasticsearch Containers",id:"to-bring-up-kibana-and-elasticsearch-containers",children:[]},{value:"For Data Saved to an NFS Volume",id:"for-data-saved-to-an-nfs-volume",children:[]}],c={toc:s};function d(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(r.b)("wrapper",Object(o.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(r.b)("p",null,"If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."),Object(r.b)("p",null,Object(r.b)("em",{parentName:"p"},"Please first read the Docker Installation guide in detail. This guide will build on top of that.")),Object(r.b)("h2",{id:"to-add-an-additional-sflow-or-netflow-collector"},"To Add an Additional Sflow or Netflow Collector"),Object(r.b)("p",null,"If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor."),Object(r.b)("p",null,"Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) "),Object(r.b)("h3",{id:"a-edit-docker-composeoverrideyml"},"a. Edit docker-compose.override.yml"),Object(r.b)("p",null,"The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},'  example-collector:\n    image: netsage/nfdump-collector:1.6.18\n    restart: always\n    command: sfcapd -T all -l /data -S 1 -w -z -p 9997\n    volumes:\n      - ./data/input_data/example:/data\n    ports:\n      - "9997:9997/udp"\n')),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},'collector-name: should be updated to something that has some meaning, in our example "example-collector".'),Object(r.b)("li",{parentName:"ul"},"command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)"),Object(r.b)("li",{parentName:"ul"},"ports: make sure the port here matches the port you've set in the command. Naturally all ports have to be unique for this host and the\nrouter should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) "),Object(r.b)("li",{parentName:"ul"},"volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we're writing to ./data/input_data/example. Change the last part of the path to something meaningful.")),Object(r.b)("p",null,"You will also need to uncomment these lines: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h3",{id:"b--edit-netsage_overridexml"},"b.  Edit netsage_override.xml"),Object(r.b)("p",null,"To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,'Edit netsage_override.xml and add a "collection" section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new "variable"; it will be replaced with a value set in the .env file. For the flow-type, enter "sflow" or "netflow" as appropriate.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"    <collection>\n        <flow-path>/data/input_data/example/</flow-path>\n        <sensor>$exampleSensorName</sensor>\n        <flow-type>sflow</flow-type>\n    </collection>\n")),Object(r.b)("h3",{id:"c-edit-environment-file"},"c. Edit environment file"),Object(r.b)("p",null,'Then, in the .env file, add a line that sets a value for the "variable" you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-ini"},"exampleSensorName=Example New York sFlow\n")),Object(r.b)("h3",{id:"d-running-the-new-collector"},"d. Running the new collector"),Object(r.b)("p",null,"After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running ",Object(r.b)("inlineCode",{parentName:"p"},"docker-compose up -d")," to start up all containers):"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"docker-compose up -d example-collector\n")),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed ",Object(r.b)("a",{parentName:"p",href:"https://hub.docker.com/r/netsage/nfdump-collector/tags"},"here")," and the source to generate the docker image can be found ",Object(r.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"here")," the code for the You may use a different version though there is no particular effort to have an image for every nfdump release."))),Object(r.b)("h2",{id:"to-keep-only-flows-from-certain-interfaces"},"To Keep Only Flows From Certain Interfaces"),Object(r.b)("p",null,"If your sensors are exporting all flows, but you only want to keep some of them (eg, only send some of them to NetSage), use this option. The collectors and importer will process all flows, but in the logstash pipeline, those that do not have src_ifindex or dst_inindex equal to one of the listed interfaces will be dropped. "),Object(r.b)("p",null,'In the .env file, uncomment the apprpriate section and enter the information required. Be sure "True" is capitalized as shown and list all the ifindex values of flows that should be kept and passed on to NetSage. You may enter one or more ifindex values. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_filter_flag=True\nifindex_filter_keep=123,456\n")),Object(r.b)("p",null,"In this case, only flows that have src_ifindex = 123 or src_ifindex = 456 or dst_ifindex = 123 or dst_ifindex = 456 will be kept. All others will be dropped."),Object(r.b)("h2",{id:"to-change-a-sensor-name-depending-on-the-interface-used"},"To Change a Sensor Name Depending on the Interface Used"),Object(r.b)("p",null,"In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 4 fields are set properly! For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"ifindex_sensor_rename_flag=True\nifindex_sensor_rename_old_name=IU Sflow \nifindex_sensor_rename_new_name=IU Bloomington Sflow\nifindex_sensor_rename_ifindex=10032\n")),Object(r.b)("p",null,'In this case, any flows from the "IU Sflow" sensor that come through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id in ElasticSearch) changed from "IU Sflow" to "IU Bloomington Sflow". Currently, only one such rename can be configured in Docker.'),Object(r.b)("div",{className:"admonition admonition-note alert alert--secondary"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated."))),Object(r.b)("h2",{id:"to-do-sampling-rate-corrections-in-logstash"},"To Do Sampling Rate Corrections in Logstash"),Object(r.b)("p",null,"When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the '-s' option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows."),Object(r.b)("p",null,'In the .env file, uncomment the appropriate section and enter the information required. Be sure "True" is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,'),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"sampling_correction_flag=True\nsampling_correction_sensors=IU Bloomington Sflow, IU Sflow\nsampling_correction_factor=512\n")),Object(r.b)("h2",{id:"to-change-how-long-nfcapd-files-are-kept"},"To Change How Long Nfcapd Files Are Kept"),Object(r.b)("p",null,"The importer will automatically delete older nfcapd files for you, so that your disk don't fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml\n")),Object(r.b)("p",null,"At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-xml"},"  <worker>\n    <cull-enable>1</cull-enable>\n    <cull-ttl>7</cull-ttl>\n  </worker>\n")),Object(r.b)("p",null,"You will also need to uncomment these lines in docker-compose.override.yml: "),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"  volumes:\n     - ./userConfig/netsage_override.xml:/etc/grnoc/netsage/deidentifier/netsage_shared.xml\n")),Object(r.b)("h2",{id:"to-customize-java-settings--increase-memory-available-for-lostash"},"To Customize Java Settings / Increase Memory Available for Lostash"),Object(r.b)("p",null,"If you need to modify the amount of memory logstash can use or any other java settings,\nrename the provided example for JVM Options and tweak the settings as desired."),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-sh"},"cp userConfig/jvm.options_example userConfig/jvm.options\n")),Object(r.b)("p",null,"Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:"),Object(r.b)("pre",null,Object(r.b)("code",{parentName:"pre",className:"language-yaml"},"logstash:\n  image: netsage/pipeline_logstash:latest\n  volumes:\n    - ./userConfig/jvm.options:/usr/share/logstash/config/jvm.options\n")),Object(r.b)("p",null,"Here are some tips for adjusting the JVM heap size (",Object(r.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"},"https://www.elastic.co/guide/en/logstash/current/jvm-settings.html"),"):"),Object(r.b)("ul",null,Object(r.b)("li",{parentName:"ul"},"The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB."),Object(r.b)("li",{parentName:"ul"},"CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves."),Object(r.b)("li",{parentName:"ul"},"Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don\u2019t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use."),Object(r.b)("li",{parentName:"ul"},"Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.")),Object(r.b)("h2",{id:"to-bring-up-kibana-and-elasticsearch-containers"},"To Bring up Kibana and Elasticsearch Containers"),Object(r.b)("p",null,"The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components."),Object(r.b)("p",null,"This isn't a production pattern but the tools can be useful at times. Please refer to the ",Object(r.b)("a",{parentName:"p",href:"../devel/docker_dev_guide#optional-elasticsearch-and-kibana"},"Docker Dev Guide")),Object(r.b)("h2",{id:"for-data-saved-to-an-nfs-volume"},"For Data Saved to an NFS Volume"),Object(r.b)("p",null,"By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either"),Object(r.b)("ol",null,Object(r.b)("li",{parentName:"ol"},"export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)"),Object(r.b)("li",{parentName:"ol"},"update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml")),Object(r.b)("p",null,"Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values."),Object(r.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(r.b)("div",{parentName:"div",className:"admonition-heading"},Object(r.b)("h5",{parentName:"div"},Object(r.b)("span",{parentName:"h5",className:"admonition-icon"},Object(r.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(r.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(r.b)("div",{parentName:"div",className:"admonition-content"},Object(r.b)("p",{parentName:"div"},"If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.\nYou'll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually."))))}d.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return p})),n.d(t,"b",(function(){return u}));var o=n(0),a=n.n(o);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=a.a.createContext({}),d=function(e){var t=a.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=d(e.components);return a.a.createElement(c.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},h=a.a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=d(n),h=o,u=p["".concat(i,".").concat(h)]||p[h]||m[h]||r;return n?a.a.createElement(u,l(l({ref:t},c),{},{components:n})):a.a.createElement(u,l({ref:t},c))}));function u(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var c=2;c<r;c++)i[c]=n[c];return a.a.createElement.apply(null,i)}return a.a.createElement.apply(null,n)}h.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/af9cfdf8.a481c25f.js b/assets/js/f2cce206.f146a973.js
similarity index 94%
rename from assets/js/af9cfdf8.a481c25f.js
rename to assets/js/f2cce206.f146a973.js
index e5fa9519..2eabc431 100644
--- a/assets/js/af9cfdf8.a481c25f.js
+++ b/assets/js/f2cce206.f146a973.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[108],{178:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(232)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-2.0.0/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-2.0.0/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/intro.md",version:"2.0.0",sidebar_label:"Intro",sidebar:"version-2.0.0/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[147],{218:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return r})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var a=n(3),i=n(7),o=(n(0),n(228)),r={id:"intro",title:"Intro",sidebar_label:"Intro"},l={unversionedId:"pipeline/intro",id:"version-2.1.0/pipeline/intro",isDocsHomePage:!1,title:"Intro",description:"The NetSage Pipeline",source:"@site/versioned_docs/version-2.1.0/pipeline/intro.md",slug:"/pipeline/intro",permalink:"/netsage-pipeline/docs/pipeline/intro",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/intro.md",version:"2.1.0",sidebar_label:"Intro",sidebar:"version-2.1.0/Pipeline",next:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/pipeline/tstat"}},s=[{value:"Description",id:"description",children:[]},{value:"Data Collection",id:"data-collection",children:[]},{value:"Pipeline Components",id:"pipeline-components",children:[]},{value:"Visualization",id:"visualization",children:[]},{value:"Pipeline Installation",id:"pipeline-installation",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(i.a)(e,["components"]);return Object(o.b)("wrapper",Object(a.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(o.b)("h1",{id:"the-netsage-pipeline"},"The NetSage Pipeline"),Object(o.b)("h2",{id:"description"},"Description"),Object(o.b)("p",null,'The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.\nThere are many ways the components can be combined, configured, and run. These documents will describe the standard "simple" set up and provide information for more complex configurations.'),Object(o.b)("h2",{id:"data-collection"},"Data Collection"),Object(o.b)("p",null,"In Netsage, sensor(s) are network devices configured to collect flow data (",Object(o.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"tstat"),", ",Object(o.b)("a",{parentName:"p",href:"https://www.rfc-editor.org/info/rfc3176"},"sflow"),", or ",Object(o.b)("a",{parentName:"p",href:"https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html"},"netflow"),') and send it to a "pipeline host" for processing. '),Object(o.b)("p",null,"Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," tool. This can be installed as usual or via Docker. "),Object(o.b)("p",null,"Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using ",Object(o.b)("a",{parentName:"p",href:"https://github.com/phaag/nfdump"},"nfdump tools"),". The Netsage project has packaged the nfdump tools into a ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"Docker container")," for ease of use."),Object(o.b)("h2",{id:"pipeline-components"},"Pipeline Components"),Object(o.b)("p",null,"The Netsage Flow Processing Pipeline is made of the following components"),Object(o.b)("ul",null,Object(o.b)("li",{parentName:"ul"},"Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/pipeline/importer"},"Doc"),", ",Object(o.b)("a",{parentName:"li",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm"},"in github"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.rabbitmq.com/"},"RabbitMQ"),": Used for message passing and queuing of tasks."),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/logstash"},"Logstash")," pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (",Object(o.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/pipeline/logstash"},"Doc"),")"),Object(o.b)("li",{parentName:"ul"},Object(o.b)("a",{parentName:"li",href:"https://www.elastic.co/what-is/elasticsearch"},"Elasticsearch"),": Used for storing the final flow data. ")),Object(o.b)("h2",{id:"visualization"},"Visualization"),Object(o.b)("p",null,Object(o.b)("a",{parentName:"p",href:"https://grafana.com/oss/grafana/"},"Grafana")," or ",Object(o.b)("a",{parentName:"p",href:"https://www.elastic.co/kibana"},"Kibana")," can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available ",Object(o.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"in github"),"."),Object(o.b)("h2",{id:"pipeline-installation"},"Pipeline Installation"),Object(o.b)("p",null,'Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the "BareMetal" or "Manual" Install). More recently, we\'ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic "Docker Installation" should suffice. The "Docker Advanced Options" guide will help when there are more sensors and/or other customizations required.'))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var a=n(0),i=n.n(a);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){o(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=i.a.createContext({}),c=function(e){var t=i.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},d=function(e){var t=c(e.components);return i.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return i.a.createElement(i.a.Fragment,{},t)}},u=i.a.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,r=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=a,f=d["".concat(r,".").concat(u)]||d[u]||b[u]||o;return n?i.a.createElement(f,l(l({ref:t},p),{},{components:n})):i.a.createElement(f,l({ref:t},p))}));function f(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=u;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,r[1]=l;for(var p=2;p<o;p++)r[p]=n[p];return i.a.createElement.apply(null,r)}return i.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f485eca7.f3f2ea52.js b/assets/js/f485eca7.3a61264f.js
similarity index 96%
rename from assets/js/f485eca7.f3f2ea52.js
rename to assets/js/f485eca7.3a61264f.js
index 39735fb8..b1070856 100644
--- a/assets/js/f485eca7.f3f2ea52.js
+++ b/assets/js/f485eca7.3a61264f.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[154],{225:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.10/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",version:"1.2.10",sidebar_label:"Replay Dataset",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[148],{219:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return c}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"version-1.2.10/devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",version:"1.2.10",sidebar_label:"Replay Dataset",sidebar:"version-1.2.10/Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"}},s=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],p={toc:s};function c(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}c.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),c=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=c(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},p),{},{components:n})):o.a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var p=2;p<a;p++)l[p]=n[p];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f52ba0ac.ba4c9a10.js b/assets/js/f52ba0ac.c7675cca.js
similarity index 95%
rename from assets/js/f52ba0ac.ba4c9a10.js
rename to assets/js/f52ba0ac.c7675cca.js
index 360caba5..b3b8adf5 100644
--- a/assets/js/f52ba0ac.ba4c9a10.js
+++ b/assets/js/f52ba0ac.c7675cca.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[155],{226:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(232)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-1.2.11/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-1.2.11/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/tstat.md",version:"1.2.11",sidebar_label:"Tstat Data",sidebar:"version-1.2.11/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[149],{220:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-1.2.11/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-1.2.11/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/tstat.md",version:"1.2.11",sidebar_label:"Tstat Data",sidebar:"version-1.2.11/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f64b3473.09cab9db.js b/assets/js/f64b3473.b96c3fd3.js
similarity index 96%
rename from assets/js/f64b3473.09cab9db.js
rename to assets/js/f64b3473.b96c3fd3.js
index 86e6f297..74f127f6 100644
--- a/assets/js/f64b3473.09cab9db.js
+++ b/assets/js/f64b3473.b96c3fd3.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[156],{227:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(232)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.9/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.9/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/choosing.md",version:"1.2.9",sidebar_label:"Choose Install",sidebar:"version-1.2.9/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[150],{221:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return a})),n.d(t,"metadata",(function(){return l})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return p}));var r=n(3),o=n(7),i=(n(0),n(228)),a={id:"choose_install",title:"Choosing Install",sidebar_label:"Choose Install"},l={unversionedId:"deploy/choose_install",id:"version-1.2.9/deploy/choose_install",isDocsHomePage:!1,title:"Choosing Install",description:"BareMetal or Server Install",source:"@site/versioned_docs/version-1.2.9/deploy/choosing.md",slug:"/deploy/choose_install",permalink:"/netsage-pipeline/docs/1.2.9/deploy/choose_install",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/choosing.md",version:"1.2.9",sidebar_label:"Choose Install",sidebar:"version-1.2.9/Pipeline",previous:{title:"Elasticsearch",permalink:"/netsage-pipeline/docs/1.2.9/pipeline/elastic"},next:{title:"NetSage Flow Processing Pipeline Installation Guide",permalink:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"}},s=[{value:"BareMetal or Server Install",id:"baremetal-or-server-install",children:[]},{value:"Dockerized Version",id:"dockerized-version",children:[]},{value:"Choose your adventure",id:"choose-your-adventure",children:[]}],c={toc:s};function p(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"baremetal-or-server-install"},"BareMetal or Server Install"),Object(i.b)("p",null,"The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved."),Object(i.b)("p",null,"It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up."),Object(i.b)("p",null,"If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it'll like need to handle a large dataset."),Object(i.b)("h2",{id:"dockerized-version"},"Dockerized Version"),Object(i.b)("p",null,"The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it's should be a simple matter of configuring a few env settings and everything should 'just' work."),Object(i.b)("p",null,"If you are simply using the pipeline to deliver the anonymized network stats for someone else's consumption, then using the docker pipeline would be preferred."),Object(i.b)("h2",{id:"choose-your-adventure"},"Choose your adventure"),Object(i.b)("ul",null,Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"bare_metal_install"},"Server Installation")),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"},"Simple Docker")," - 1 netflow sensor and/or 1 sflow sensor"),Object(i.b)("li",{parentName:"ul"},Object(i.b)("a",{parentName:"li",href:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"},"Advanced Docker")," - allows for more complex configurations")))}p.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?a(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):a(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),p=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,a=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(n),b=r,m=u["".concat(a,".").concat(b)]||u[b]||d[b]||i;return n?o.a.createElement(m,l(l({ref:t},c),{},{components:n})):o.a.createElement(m,l({ref:t},c))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=b;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,a[1]=l;for(var c=2;c<i;c++)a[c]=n[c];return o.a.createElement.apply(null,a)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f8bc0717.28c2f6ec.js b/assets/js/f8bc0717.e9c8f77e.js
similarity index 96%
rename from assets/js/f8bc0717.28c2f6ec.js
rename to assets/js/f8bc0717.e9c8f77e.js
index bf328972..958cb67e 100644
--- a/assets/js/f8bc0717.28c2f6ec.js
+++ b/assets/js/f8bc0717.e9c8f77e.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[157],{228:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(232)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-1.2.10/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.10/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/nfdump.md",version:"1.2.10",sidebar_label:"Sflow/Netflow Data",sidebar:"version-1.2.10/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[151],{222:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return p})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={id:"nfdump",title:"Sflow/Netflow Data Collection",sidebar_label:"Sflow/Netflow Data"},p={unversionedId:"pipeline/nfdump",id:"version-1.2.10/pipeline/nfdump",isDocsHomePage:!1,title:"Sflow/Netflow Data Collection",description:"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.",source:"@site/versioned_docs/version-1.2.10/pipeline/nfdump.md",slug:"/pipeline/nfdump",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/nfdump.md",version:"1.2.10",sidebar_label:"Sflow/Netflow Data",sidebar:"version-1.2.10/Pipeline",previous:{title:"Tstat Data Collection",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/tstat"},next:{title:"Importer",permalink:"/netsage-pipeline/docs/1.2.10/pipeline/importer"}},c=[{value:"Netsage Usage",id:"netsage-usage",children:[]},{value:"Docker Deployment",id:"docker-deployment",children:[]}],s={toc:c};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},s,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."),Object(a.b)("h2",{id:"netsage-usage"},"Netsage Usage"),Object(a.b)("p",null,"Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the ",Object(a.b)("a",{parentName:"p",href:"importer"},"importer"),", which uses an nfdump command, and sent to RabbitMQ. From there, the ",Object(a.b)("a",{parentName:"p",href:"logstash"},"logstash")," pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"grafana dashboards"),"."),Object(a.b)("p",null,"One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window."),Object(a.b)("h2",{id:"docker-deployment"},"Docker Deployment"),Object(a.b)("p",null,"The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/docker-nfdump-collector"},"HERE")))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return m}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function p(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=o.a.createContext({}),l=function(e){var t=o.a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):p(p({},t),e)),n},d=function(e){var t=l(e.components);return o.a.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},u=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,s=c(e,["components","mdxType","originalType","parentName"]),d=l(n),u=r,m=d["".concat(i,".").concat(u)]||d[u]||f[u]||a;return n?o.a.createElement(m,p(p({ref:t},s),{},{components:n})):o.a.createElement(m,p({ref:t},s))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=u;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s<a;s++)i[s]=n[s];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}u.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/f9995f72.b29de3ff.js b/assets/js/f9995f72.b29de3ff.js
new file mode 100644
index 00000000..ac6b6d87
--- /dev/null
+++ b/assets/js/f9995f72.b29de3ff.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[152],{223:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return o})),n.d(t,"metadata",(function(){return s})),n.d(t,"toc",(function(){return c})),n.d(t,"default",(function(){return l}));var r=n(3),a=n(7),i=(n(0),n(228)),o={id:"tstat",title:"Tstat Data Collection",sidebar_label:"Tstat Data"},s={unversionedId:"pipeline/tstat",id:"version-2.1.0/pipeline/tstat",isDocsHomePage:!1,title:"Tstat Data Collection",description:"Netsage GitHub Project",source:"@site/versioned_docs/version-2.1.0/pipeline/tstat.md",slug:"/pipeline/tstat",permalink:"/netsage-pipeline/docs/pipeline/tstat",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/tstat.md",version:"2.1.0",sidebar_label:"Tstat Data",sidebar:"version-2.1.0/Pipeline",previous:{title:"Intro",permalink:"/netsage-pipeline/docs/pipeline/intro"},next:{title:"Sflow/Netflow Data Collection",permalink:"/netsage-pipeline/docs/pipeline/nfdump"}},c=[{value:"Netsage GitHub Project",id:"netsage-github-project",children:[]},{value:"Docker",id:"docker",children:[]}],p={toc:c};function l(e){var t=e.components,n=Object(a.a)(e,["components"]);return Object(i.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(i.b)("h2",{id:"netsage-github-project"},"Netsage GitHub Project"),Object(i.b)("p",null,Object(i.b)("a",{parentName:"p",href:"http://tstat.polito.it/"},"Tstat")," is a passive sniffer that provides insights into traffic patterns.  The Netsage ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport"},"tstat-transport")," project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the ",Object(i.b)("a",{parentName:"p",href:"logstash"},"logstash pipeline"),", stored in elasticsearch, and finally displayed in our Grafana ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-grafana-configs"},"dashboards"),"."),Object(i.b)("h2",{id:"docker"},"Docker"),Object(i.b)("p",null,"Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available ",Object(i.b)("a",{parentName:"p",href:"https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md"},"here"),".  "))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return f}));var r=n(0),a=n.n(r);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function c(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)n=i[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=a.a.createContext({}),l=function(e){var t=a.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return a.a.createElement(p.Provider,{value:t},e.children)},b={inlineCode:"code",wrapper:function(e){var t=e.children;return a.a.createElement(a.a.Fragment,{},t)}},d=a.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,o=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),u=l(n),d=r,f=u["".concat(o,".").concat(d)]||u[d]||b[d]||i;return n?a.a.createElement(f,s(s({ref:t},p),{},{components:n})):a.a.createElement(f,s({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:r,o[1]=s;for(var p=2;p<i;p++)o[p]=n[p];return a.a.createElement.apply(null,o)}return a.a.createElement.apply(null,n)}d.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/fa16f0fc.a192e853.js b/assets/js/fa16f0fc.8db267a7.js
similarity index 96%
rename from assets/js/fa16f0fc.a192e853.js
rename to assets/js/fa16f0fc.8db267a7.js
index de832691..c3063c3f 100644
--- a/assets/js/fa16f0fc.a192e853.js
+++ b/assets/js/fa16f0fc.8db267a7.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[158],{229:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(232)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/docs/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/pipeline_dataset.md",version:"current",sidebar_label:"Replay Dataset",sidebar:"Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide"}},p=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],c={toc:p};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/next/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}s.isMDXComponent=!0},232:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),s=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},c),{},{components:n})):o.a.createElement(f,i({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var c=2;c<a;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[153],{224:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return l})),n.d(t,"metadata",(function(){return i})),n.d(t,"toc",(function(){return p})),n.d(t,"default",(function(){return s}));var r=n(3),o=n(7),a=(n(0),n(228)),l={id:"dev_dataset",title:"Pipeline Replay Dataset",sidebar_label:"Replay Dataset"},i={unversionedId:"devel/dev_dataset",id:"devel/dev_dataset",isDocsHomePage:!1,title:"Pipeline Replay Dataset",description:"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test",source:"@site/docs/devel/pipeline_dataset.md",slug:"/devel/dev_dataset",permalink:"/netsage-pipeline/docs/next/devel/dev_dataset",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/pipeline_dataset.md",version:"current",sidebar_label:"Replay Dataset",sidebar:"Pipeline",previous:{title:"Docker Troubleshooting",permalink:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot"},next:{title:"Docker Dev Guide",permalink:"/netsage-pipeline/docs/next/devel/docker_dev_guide"}},p=[{value:"Netflow",id:"netflow",children:[]},{value:"Sflow",id:"sflow",children:[]}],c={toc:p};function s(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},c,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("p",null,"The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test\nthe behavior using things like the Ruby unit ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec"},"tests")," in logstash, and the ",Object(a.b)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html"},"generator")," pligin, but the best\ntest is to replay network data and inspect the output in the grafana dashboard. "),Object(a.b)("p",null,"Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. "),Object(a.b)("p",null,"You can download the files from ",Object(a.b)("a",{parentName:"p",href:"https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG"},"here"),"."),Object(a.b)("p",null,"Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this ",Object(a.b)("a",{parentName:"p",href:"https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml"},"example"),". "),Object(a.b)("p",null,"Currently the default ports are:"),Object(a.b)("ul",null,Object(a.b)("li",{parentName:"ul"},"9998/udp for sflow "),Object(a.b)("li",{parentName:"ul"},"9999/udp for netflow")),Object(a.b)("p",null,"Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the ",Object(a.b)("a",{parentName:"p",href:"/netsage-pipeline/docs/next/deploy/docker_install_simple#running-the-collectors"},"Docker Simple Deployment Guide"),"  "),Object(a.b)("p",null,"In order to replay the data, use the following commands for netflow and sflow respectively:"),Object(a.b)("h3",{id:"netflow"},"Netflow"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre"},"nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000\n")),Object(a.b)("h3",{id:"sflow"},"Sflow"),Object(a.b)("p",null,"Coming soon.  nfreplay will not work with sflow data type."))}s.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return d})),n.d(t,"b",(function(){return f}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?l(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):l(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=o.a.createContext({}),s=function(e){var t=o.a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=s(e.components);return o.a.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},b=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=s(n),b=r,f=d["".concat(l,".").concat(b)]||d[b]||u[b]||a;return n?o.a.createElement(f,i(i({ref:t},c),{},{components:n})):o.a.createElement(f,i({ref:t},c))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,l=new Array(a);l[0]=b;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:r,l[1]=i;for(var c=2;c<a;c++)l[c]=n[c];return o.a.createElement.apply(null,l)}return o.a.createElement.apply(null,n)}b.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/ff1d9e71.c2bc6db2.js b/assets/js/ff1d9e71.c2bc6db2.js
new file mode 100644
index 00000000..94bd3943
--- /dev/null
+++ b/assets/js/ff1d9e71.c2bc6db2.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[154],{225:function(e,t,n){"use strict";n.r(t),n.d(t,"frontMatter",(function(){return i})),n.d(t,"metadata",(function(){return c})),n.d(t,"toc",(function(){return s})),n.d(t,"default",(function(){return l}));var r=n(3),o=n(7),a=(n(0),n(228)),i={},c={unversionedId:"components/docker_first_steps",id:"version-2.1.0/components/docker_first_steps",isDocsHomePage:!1,title:"docker_first_steps",description:"saving this for now in case I need to put it back #",source:"@site/versioned_docs/version-2.1.0/components/docker_first_steps.md",slug:"/components/docker_first_steps",permalink:"/netsage-pipeline/docs/components/docker_first_steps",editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_first_steps.md",version:"2.1.0"},s=[],p={toc:s};function l(e){var t=e.components,n=Object(o.a)(e,["components"]);return Object(a.b)("wrapper",Object(r.a)({},p,n,{components:t,mdxType:"MDXLayout"}),Object(a.b)("h4",{id:"saving-this-for-now-in-case-i-need-to-put-it-back"},"saving this for now in case I need to put it back"),Object(a.b)("p",null,"Then checkout the latest version of the code.  If you are a developer you'll want the latest version from master, otherwise please use make sure\nyou've checked out the latest tagged version."),Object(a.b)("p",null,"For example,"),Object(a.b)("pre",null,Object(a.b)("code",{parentName:"pre",className:"language-sh"},"## Normal Deployment, eg, checkout version 1.2.8\n$ git fetch \n$ git checkout v1.2.8 -b v1.2.8\n\n## Developers\n$ git fetch \n$ git reset --hard origin/master\n")),Object(a.b)("div",{className:"admonition admonition-warning alert alert--danger"},Object(a.b)("div",{parentName:"div",className:"admonition-heading"},Object(a.b)("h5",{parentName:"div"},Object(a.b)("span",{parentName:"h5",className:"admonition-icon"},Object(a.b)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"12",height:"16",viewBox:"0 0 12 16"},Object(a.b)("path",{parentName:"svg",fillRule:"evenodd",d:"M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"}))),"warning")),Object(a.b)("div",{parentName:"div",className:"admonition-content"},Object(a.b)("p",{parentName:"div"},"git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing"),Object(a.b)("p",{parentName:"div"},"Example:\n",Object(a.b)("inlineCode",{parentName:"p"},'git commit -a -m "Saving local state"; git checkout -b feature/backup; git checkout master')))),Object(a.b)("p",null,"All instructions that follow assume these first steps were performed succesfully.  If not, you'll likely run into errors down the line if the code doesn't match up with the instructions provided."))}l.isMDXComponent=!0},228:function(e,t,n){"use strict";n.d(t,"a",(function(){return u})),n.d(t,"b",(function(){return b}));var r=n(0),o=n.n(r);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r<a.length;r++)n=a[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=o.a.createContext({}),l=function(e){var t=o.a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=l(e.components);return o.a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.a.createElement(o.a.Fragment,{},t)}},m=o.a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(n),m=r,b=u["".concat(i,".").concat(m)]||u[m]||d[m]||a;return n?o.a.createElement(b,c(c({ref:t},p),{},{components:n})):o.a.createElement(b,c({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c.mdxType="string"==typeof e?e:r,i[1]=c;for(var p=2;p<a;p++)i[p]=n[p];return o.a.createElement.apply(null,i)}return o.a.createElement.apply(null,n)}m.displayName="MDXCreateElement"}}]);
\ No newline at end of file
diff --git a/assets/js/main.16ba1fad.js b/assets/js/main.16ba1fad.js
new file mode 100644
index 00000000..5d954ac8
--- /dev/null
+++ b/assets/js/main.16ba1fad.js
@@ -0,0 +1,2 @@
+/*! For license information please see main.16ba1fad.js.LICENSE.txt */
+(window.webpackJsonp=window.webpackJsonp||[]).push([[155],[function(e,n,t){"use strict";e.exports=t(39)},function(e,n,t){e.exports=t(43)()},function(e,n,t){"use strict";t.d(n,"a",(function(){return b})),t.d(n,"b",(function(){return k})),t.d(n,"c",(function(){return E})),t.d(n,"d",(function(){return C})),t.d(n,"e",(function(){return v})),t.d(n,"f",(function(){return L})),t.d(n,"g",(function(){return D})),t.d(n,"h",(function(){return h})),t.d(n,"i",(function(){return x})),t.d(n,"j",(function(){return P})),t.d(n,"k",(function(){return F})),t.d(n,"l",(function(){return B})),t.d(n,"m",(function(){return z})),t.d(n,"n",(function(){return U})),t.d(n,"o",(function(){return j}));var i=t(4),r=t(0),o=t.n(r),a=(t(1),t(6)),l=t(21),s=t(5),c=t(3),u=t(22),p=t.n(u),d=(t(29),t(7)),f=t(34),m=t.n(f),g=function(e){var n=Object(l.a)();return n.displayName=e,n}("Router-History"),h=function(e){var n=Object(l.a)();return n.displayName=e,n}("Router"),v=function(e){function n(n){var t;return(t=e.call(this,n)||this).state={location:n.history.location},t._isMounted=!1,t._pendingLocation=null,n.staticContext||(t.unlisten=n.history.listen((function(e){t._isMounted?t.setState({location:e}):t._pendingLocation=e}))),t}Object(i.a)(n,e),n.computeRootMatch=function(e){return{path:"/",url:"/",params:{},isExact:"/"===e}};var t=n.prototype;return t.componentDidMount=function(){this._isMounted=!0,this._pendingLocation&&this.setState({location:this._pendingLocation})},t.componentWillUnmount=function(){this.unlisten&&this.unlisten()},t.render=function(){return o.a.createElement(h.Provider,{value:{history:this.props.history,location:this.state.location,match:n.computeRootMatch(this.state.location.pathname),staticContext:this.props.staticContext}},o.a.createElement(g.Provider,{children:this.props.children||null,value:this.props.history}))},n}(o.a.Component);var b=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(a.d)(n.props),n}return Object(i.a)(n,e),n.prototype.render=function(){return o.a.createElement(v,{history:this.history,children:this.props.children})},n}(o.a.Component);var y=function(e){function n(){return e.apply(this,arguments)||this}Object(i.a)(n,e);var t=n.prototype;return t.componentDidMount=function(){this.props.onMount&&this.props.onMount.call(this,this)},t.componentDidUpdate=function(e){this.props.onUpdate&&this.props.onUpdate.call(this,this,e)},t.componentWillUnmount=function(){this.props.onUnmount&&this.props.onUnmount.call(this,this)},t.render=function(){return null},n}(o.a.Component);function k(e){var n=e.message,t=e.when,i=void 0===t||t;return o.a.createElement(h.Consumer,null,(function(e){if(e||Object(s.a)(!1),!i||e.staticContext)return null;var t=e.history.block;return o.a.createElement(y,{onMount:function(e){e.release=t(n)},onUpdate:function(e,i){i.message!==n&&(e.release(),e.release=t(n))},onUnmount:function(e){e.release()},message:n})}))}var w={},_=0;function x(e,n){return void 0===e&&(e="/"),void 0===n&&(n={}),"/"===e?e:function(e){if(w[e])return w[e];var n=p.a.compile(e);return _<1e4&&(w[e]=n,_++),n}(e)(n,{pretty:!0})}function E(e){var n=e.computedMatch,t=e.to,i=e.push,r=void 0!==i&&i;return o.a.createElement(h.Consumer,null,(function(e){e||Object(s.a)(!1);var i=e.history,l=e.staticContext,u=r?i.push:i.replace,p=Object(a.c)(n?"string"==typeof t?x(t,n.params):Object(c.a)({},t,{pathname:x(t.pathname,n.params)}):t);return l?(u(p),null):o.a.createElement(y,{onMount:function(){u(p)},onUpdate:function(e,n){var t=Object(a.c)(n.to);Object(a.f)(t,Object(c.a)({},p,{key:t.key}))||u(p)},to:t})}))}var S={},T=0;function P(e,n){void 0===n&&(n={}),("string"==typeof n||Array.isArray(n))&&(n={path:n});var t=n,i=t.path,r=t.exact,o=void 0!==r&&r,a=t.strict,l=void 0!==a&&a,s=t.sensitive,c=void 0!==s&&s;return[].concat(i).reduce((function(n,t){if(!t&&""!==t)return null;if(n)return n;var i=function(e,n){var t=""+n.end+n.strict+n.sensitive,i=S[t]||(S[t]={});if(i[e])return i[e];var r=[],o={regexp:p()(e,r,n),keys:r};return T<1e4&&(i[e]=o,T++),o}(t,{end:o,strict:l,sensitive:c}),r=i.regexp,a=i.keys,s=r.exec(e);if(!s)return null;var u=s[0],d=s.slice(1),f=e===u;return o&&!f?null:{path:t,url:"/"===t&&""===u?"/":u,isExact:f,params:a.reduce((function(e,n,t){return e[n.name]=d[t],e}),{})}}),null)}var C=function(e){function n(){return e.apply(this,arguments)||this}return Object(i.a)(n,e),n.prototype.render=function(){var e=this;return o.a.createElement(h.Consumer,null,(function(n){n||Object(s.a)(!1);var t=e.props.location||n.location,i=e.props.computedMatch?e.props.computedMatch:e.props.path?P(t.pathname,e.props):n.match,r=Object(c.a)({},n,{location:t,match:i}),a=e.props,l=a.children,u=a.component,p=a.render;return Array.isArray(l)&&0===l.length&&(l=null),o.a.createElement(h.Provider,{value:r},r.match?l?"function"==typeof l?l(r):l:u?o.a.createElement(u,r):p?p(r):null:"function"==typeof l?l(r):null)}))},n}(o.a.Component);function O(e){return"/"===e.charAt(0)?e:"/"+e}function A(e,n){if(!e)return n;var t=O(e);return 0!==n.pathname.indexOf(t)?n:Object(c.a)({},n,{pathname:n.pathname.substr(t.length)})}function R(e){return"string"==typeof e?e:Object(a.e)(e)}function N(e){return function(){Object(s.a)(!1)}}function I(){}var L=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).handlePush=function(e){return n.navigateTo(e,"PUSH")},n.handleReplace=function(e){return n.navigateTo(e,"REPLACE")},n.handleListen=function(){return I},n.handleBlock=function(){return I},n}Object(i.a)(n,e);var t=n.prototype;return t.navigateTo=function(e,n){var t=this.props,i=t.basename,r=void 0===i?"":i,o=t.context,l=void 0===o?{}:o;l.action=n,l.location=function(e,n){return e?Object(c.a)({},n,{pathname:O(e)+n.pathname}):n}(r,Object(a.c)(e)),l.url=R(l.location)},t.render=function(){var e=this.props,n=e.basename,t=void 0===n?"":n,i=e.context,r=void 0===i?{}:i,l=e.location,s=void 0===l?"/":l,u=Object(d.a)(e,["basename","context","location"]),p={createHref:function(e){return O(t+R(e))},action:"POP",location:A(t,Object(a.c)(s)),push:this.handlePush,replace:this.handleReplace,go:N(),goBack:N(),goForward:N(),listen:this.handleListen,block:this.handleBlock};return o.a.createElement(v,Object(c.a)({},u,{history:p,staticContext:r}))},n}(o.a.Component);var D=function(e){function n(){return e.apply(this,arguments)||this}return Object(i.a)(n,e),n.prototype.render=function(){var e=this;return o.a.createElement(h.Consumer,null,(function(n){n||Object(s.a)(!1);var t,i,r=e.props.location||n.location;return o.a.Children.forEach(e.props.children,(function(e){if(null==i&&o.a.isValidElement(e)){t=e;var a=e.props.path||e.props.from;i=a?P(r.pathname,Object(c.a)({},e.props,{path:a})):n.match}})),i?o.a.cloneElement(t,{location:r,computedMatch:i}):null}))},n}(o.a.Component);function j(e){var n="withRouter("+(e.displayName||e.name)+")",t=function(n){var t=n.wrappedComponentRef,i=Object(d.a)(n,["wrappedComponentRef"]);return o.a.createElement(h.Consumer,null,(function(n){return n||Object(s.a)(!1),o.a.createElement(e,Object(c.a)({},i,n,{ref:t}))}))};return t.displayName=n,t.WrappedComponent=e,m()(t,e)}var M=o.a.useContext;function F(){return M(g)}function B(){return M(h).location}function z(){var e=M(h).match;return e?e.params:{}}function U(e){var n=B(),t=M(h).match;return e?P(n.pathname,e):t}},function(e,n,t){"use strict";function i(){return(i=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e}).apply(this,arguments)}t.d(n,"a",(function(){return i}))},function(e,n,t){"use strict";function i(e,n){return(i=Object.setPrototypeOf||function(e,n){return e.__proto__=n,e})(e,n)}function r(e,n){e.prototype=Object.create(n.prototype),e.prototype.constructor=e,i(e,n)}t.d(n,"a",(function(){return r}))},function(e,n,t){"use strict";var i="Invariant failed";n.a=function(e,n){if(!e)throw new Error(i)}},function(e,n,t){"use strict";t.d(n,"a",(function(){return x})),t.d(n,"b",(function(){return O})),t.d(n,"d",(function(){return R})),t.d(n,"c",(function(){return g})),t.d(n,"f",(function(){return h})),t.d(n,"e",(function(){return m}));var i=t(3);function r(e){return"/"===e.charAt(0)}function o(e,n){for(var t=n,i=t+1,r=e.length;i<r;t+=1,i+=1)e[t]=e[i];e.pop()}var a=function(e,n){void 0===n&&(n="");var t,i=e&&e.split("/")||[],a=n&&n.split("/")||[],l=e&&r(e),s=n&&r(n),c=l||s;if(e&&r(e)?a=i:i.length&&(a.pop(),a=a.concat(i)),!a.length)return"/";if(a.length){var u=a[a.length-1];t="."===u||".."===u||""===u}else t=!1;for(var p=0,d=a.length;d>=0;d--){var f=a[d];"."===f?o(a,d):".."===f?(o(a,d),p++):p&&(o(a,d),p--)}if(!c)for(;p--;p)a.unshift("..");!c||""===a[0]||a[0]&&r(a[0])||a.unshift("");var m=a.join("/");return t&&"/"!==m.substr(-1)&&(m+="/"),m};function l(e){return e.valueOf?e.valueOf():Object.prototype.valueOf.call(e)}var s=function e(n,t){if(n===t)return!0;if(null==n||null==t)return!1;if(Array.isArray(n))return Array.isArray(t)&&n.length===t.length&&n.every((function(n,i){return e(n,t[i])}));if("object"==typeof n||"object"==typeof t){var i=l(n),r=l(t);return i!==n||r!==t?e(i,r):Object.keys(Object.assign({},n,t)).every((function(i){return e(n[i],t[i])}))}return!1},c=t(5);function u(e){return"/"===e.charAt(0)?e:"/"+e}function p(e){return"/"===e.charAt(0)?e.substr(1):e}function d(e,n){return function(e,n){return 0===e.toLowerCase().indexOf(n.toLowerCase())&&-1!=="/?#".indexOf(e.charAt(n.length))}(e,n)?e.substr(n.length):e}function f(e){return"/"===e.charAt(e.length-1)?e.slice(0,-1):e}function m(e){var n=e.pathname,t=e.search,i=e.hash,r=n||"/";return t&&"?"!==t&&(r+="?"===t.charAt(0)?t:"?"+t),i&&"#"!==i&&(r+="#"===i.charAt(0)?i:"#"+i),r}function g(e,n,t,r){var o;"string"==typeof e?(o=function(e){var n=e||"/",t="",i="",r=n.indexOf("#");-1!==r&&(i=n.substr(r),n=n.substr(0,r));var o=n.indexOf("?");return-1!==o&&(t=n.substr(o),n=n.substr(0,o)),{pathname:n,search:"?"===t?"":t,hash:"#"===i?"":i}}(e)).state=n:(void 0===(o=Object(i.a)({},e)).pathname&&(o.pathname=""),o.search?"?"!==o.search.charAt(0)&&(o.search="?"+o.search):o.search="",o.hash?"#"!==o.hash.charAt(0)&&(o.hash="#"+o.hash):o.hash="",void 0!==n&&void 0===o.state&&(o.state=n));try{o.pathname=decodeURI(o.pathname)}catch(l){throw l instanceof URIError?new URIError('Pathname "'+o.pathname+'" could not be decoded. This is likely caused by an invalid percent-encoding.'):l}return t&&(o.key=t),r?o.pathname?"/"!==o.pathname.charAt(0)&&(o.pathname=a(o.pathname,r.pathname)):o.pathname=r.pathname:o.pathname||(o.pathname="/"),o}function h(e,n){return e.pathname===n.pathname&&e.search===n.search&&e.hash===n.hash&&e.key===n.key&&s(e.state,n.state)}function v(){var e=null;var n=[];return{setPrompt:function(n){return e=n,function(){e===n&&(e=null)}},confirmTransitionTo:function(n,t,i,r){if(null!=e){var o="function"==typeof e?e(n,t):e;"string"==typeof o?"function"==typeof i?i(o,r):r(!0):r(!1!==o)}else r(!0)},appendListener:function(e){var t=!0;function i(){t&&e.apply(void 0,arguments)}return n.push(i),function(){t=!1,n=n.filter((function(e){return e!==i}))}},notifyListeners:function(){for(var e=arguments.length,t=new Array(e),i=0;i<e;i++)t[i]=arguments[i];n.forEach((function(e){return e.apply(void 0,t)}))}}}var b=!("undefined"==typeof window||!window.document||!window.document.createElement);function y(e,n){n(window.confirm(e))}var k="popstate",w="hashchange";function _(){try{return window.history.state||{}}catch(e){return{}}}function x(e){void 0===e&&(e={}),b||Object(c.a)(!1);var n,t=window.history,r=(-1===(n=window.navigator.userAgent).indexOf("Android 2.")&&-1===n.indexOf("Android 4.0")||-1===n.indexOf("Mobile Safari")||-1!==n.indexOf("Chrome")||-1!==n.indexOf("Windows Phone"))&&window.history&&"pushState"in window.history,o=!(-1===window.navigator.userAgent.indexOf("Trident")),a=e,l=a.forceRefresh,s=void 0!==l&&l,p=a.getUserConfirmation,h=void 0===p?y:p,x=a.keyLength,E=void 0===x?6:x,S=e.basename?f(u(e.basename)):"";function T(e){var n=e||{},t=n.key,i=n.state,r=window.location,o=r.pathname+r.search+r.hash;return S&&(o=d(o,S)),g(o,i,t)}function P(){return Math.random().toString(36).substr(2,E)}var C=v();function O(e){Object(i.a)(U,e),U.length=t.length,C.notifyListeners(U.location,U.action)}function A(e){(function(e){return void 0===e.state&&-1===navigator.userAgent.indexOf("CriOS")})(e)||I(T(e.state))}function R(){I(T(_()))}var N=!1;function I(e){if(N)N=!1,O();else{C.confirmTransitionTo(e,"POP",h,(function(n){n?O({action:"POP",location:e}):function(e){var n=U.location,t=D.indexOf(n.key);-1===t&&(t=0);var i=D.indexOf(e.key);-1===i&&(i=0);var r=t-i;r&&(N=!0,M(r))}(e)}))}}var L=T(_()),D=[L.key];function j(e){return S+m(e)}function M(e){t.go(e)}var F=0;function B(e){1===(F+=e)&&1===e?(window.addEventListener(k,A),o&&window.addEventListener(w,R)):0===F&&(window.removeEventListener(k,A),o&&window.removeEventListener(w,R))}var z=!1;var U={length:t.length,action:"POP",location:L,createHref:j,push:function(e,n){var i="PUSH",o=g(e,n,P(),U.location);C.confirmTransitionTo(o,i,h,(function(e){if(e){var n=j(o),a=o.key,l=o.state;if(r)if(t.pushState({key:a,state:l},null,n),s)window.location.href=n;else{var c=D.indexOf(U.location.key),u=D.slice(0,c+1);u.push(o.key),D=u,O({action:i,location:o})}else window.location.href=n}}))},replace:function(e,n){var i="REPLACE",o=g(e,n,P(),U.location);C.confirmTransitionTo(o,i,h,(function(e){if(e){var n=j(o),a=o.key,l=o.state;if(r)if(t.replaceState({key:a,state:l},null,n),s)window.location.replace(n);else{var c=D.indexOf(U.location.key);-1!==c&&(D[c]=o.key),O({action:i,location:o})}else window.location.replace(n)}}))},go:M,goBack:function(){M(-1)},goForward:function(){M(1)},block:function(e){void 0===e&&(e=!1);var n=C.setPrompt(e);return z||(B(1),z=!0),function(){return z&&(z=!1,B(-1)),n()}},listen:function(e){var n=C.appendListener(e);return B(1),function(){B(-1),n()}}};return U}var E="hashchange",S={hashbang:{encodePath:function(e){return"!"===e.charAt(0)?e:"!/"+p(e)},decodePath:function(e){return"!"===e.charAt(0)?e.substr(1):e}},noslash:{encodePath:p,decodePath:u},slash:{encodePath:u,decodePath:u}};function T(e){var n=e.indexOf("#");return-1===n?e:e.slice(0,n)}function P(){var e=window.location.href,n=e.indexOf("#");return-1===n?"":e.substring(n+1)}function C(e){window.location.replace(T(window.location.href)+"#"+e)}function O(e){void 0===e&&(e={}),b||Object(c.a)(!1);var n=window.history,t=(window.navigator.userAgent.indexOf("Firefox"),e),r=t.getUserConfirmation,o=void 0===r?y:r,a=t.hashType,l=void 0===a?"slash":a,s=e.basename?f(u(e.basename)):"",p=S[l],h=p.encodePath,k=p.decodePath;function w(){var e=k(P());return s&&(e=d(e,s)),g(e)}var _=v();function x(e){Object(i.a)(z,e),z.length=n.length,_.notifyListeners(z.location,z.action)}var O=!1,A=null;function R(){var e,n,t=P(),i=h(t);if(t!==i)C(i);else{var r=w(),a=z.location;if(!O&&(n=r,(e=a).pathname===n.pathname&&e.search===n.search&&e.hash===n.hash))return;if(A===m(r))return;A=null,function(e){if(O)O=!1,x();else{var n="POP";_.confirmTransitionTo(e,n,o,(function(t){t?x({action:n,location:e}):function(e){var n=z.location,t=D.lastIndexOf(m(n));-1===t&&(t=0);var i=D.lastIndexOf(m(e));-1===i&&(i=0);var r=t-i;r&&(O=!0,j(r))}(e)}))}}(r)}}var N=P(),I=h(N);N!==I&&C(I);var L=w(),D=[m(L)];function j(e){n.go(e)}var M=0;function F(e){1===(M+=e)&&1===e?window.addEventListener(E,R):0===M&&window.removeEventListener(E,R)}var B=!1;var z={length:n.length,action:"POP",location:L,createHref:function(e){var n=document.querySelector("base"),t="";return n&&n.getAttribute("href")&&(t=T(window.location.href)),t+"#"+h(s+m(e))},push:function(e,n){var t="PUSH",i=g(e,void 0,void 0,z.location);_.confirmTransitionTo(i,t,o,(function(e){if(e){var n=m(i),r=h(s+n);if(P()!==r){A=n,function(e){window.location.hash=e}(r);var o=D.lastIndexOf(m(z.location)),a=D.slice(0,o+1);a.push(n),D=a,x({action:t,location:i})}else x()}}))},replace:function(e,n){var t="REPLACE",i=g(e,void 0,void 0,z.location);_.confirmTransitionTo(i,t,o,(function(e){if(e){var n=m(i),r=h(s+n);P()!==r&&(A=n,C(r));var o=D.indexOf(m(z.location));-1!==o&&(D[o]=n),x({action:t,location:i})}}))},go:j,goBack:function(){j(-1)},goForward:function(){j(1)},block:function(e){void 0===e&&(e=!1);var n=_.setPrompt(e);return B||(F(1),B=!0),function(){return B&&(B=!1,F(-1)),n()}},listen:function(e){var n=_.appendListener(e);return F(1),function(){F(-1),n()}}};return z}function A(e,n,t){return Math.min(Math.max(e,n),t)}function R(e){void 0===e&&(e={});var n=e,t=n.getUserConfirmation,r=n.initialEntries,o=void 0===r?["/"]:r,a=n.initialIndex,l=void 0===a?0:a,s=n.keyLength,c=void 0===s?6:s,u=v();function p(e){Object(i.a)(k,e),k.length=k.entries.length,u.notifyListeners(k.location,k.action)}function d(){return Math.random().toString(36).substr(2,c)}var f=A(l,0,o.length-1),h=o.map((function(e){return g(e,void 0,"string"==typeof e?d():e.key||d())})),b=m;function y(e){var n=A(k.index+e,0,k.entries.length-1),i=k.entries[n];u.confirmTransitionTo(i,"POP",t,(function(e){e?p({action:"POP",location:i,index:n}):p()}))}var k={length:h.length,action:"POP",location:h[f],index:f,entries:h,createHref:b,push:function(e,n){var i="PUSH",r=g(e,n,d(),k.location);u.confirmTransitionTo(r,i,t,(function(e){if(e){var n=k.index+1,t=k.entries.slice(0);t.length>n?t.splice(n,t.length-n,r):t.push(r),p({action:i,location:r,index:n,entries:t})}}))},replace:function(e,n){var i="REPLACE",r=g(e,n,d(),k.location);u.confirmTransitionTo(r,i,t,(function(e){e&&(k.entries[k.index]=r,p({action:i,location:r}))}))},go:y,goBack:function(){y(-1)},goForward:function(){y(1)},canGo:function(e){var n=k.index+e;return n>=0&&n<k.entries.length},block:function(e){return void 0===e&&(e=!1),u.setPrompt(e)},listen:function(e){return u.appendListener(e)}};return k}},function(e,n,t){"use strict";function i(e,n){if(null==e)return{};var t,i,r={},o=Object.keys(e);for(i=0;i<o.length;i++)t=o[i],n.indexOf(t)>=0||(r[t]=e[t]);return r}t.d(n,"a",(function(){return i}))},function(e,n,t){"use strict";var i=!("undefined"==typeof window||!window.document||!window.document.createElement),r={canUseDOM:i,canUseEventListeners:i&&!(!window.addEventListener&&!window.attachEvent),canUseIntersectionObserver:i&&"IntersectionObserver"in window,canUseViewport:i&&!!window.screen};n.a=r},function(e,n,t){"use strict";t.d(n,"a",(function(){return l})),t.d(n,"b",(function(){return s}));var i=t(2),r=t(3),o=t(0),a=t.n(o);function l(e,n,t){return void 0===t&&(t=[]),e.some((function(e){var r=e.path?Object(i.j)(n,e):t.length?t[t.length-1].match:i.e.computeRootMatch(n);return r&&(t.push({route:e,match:r}),e.routes&&l(e.routes,n,t)),r})),t}function s(e,n,t){return void 0===n&&(n={}),void 0===t&&(t={}),e?a.a.createElement(i.g,t,e.map((function(e,t){return a.a.createElement(i.d,{key:e.key||t,path:e.path,exact:e.exact,strict:e.strict,render:function(t){return e.render?e.render(Object(r.a)({},t,{},n,{route:e})):a.a.createElement(e.component,Object(r.a)({},t,n,{route:e}))}})}))):null}},function(e,n,t){"use strict";t.d(n,"a",(function(){return p})),t.d(n,"b",(function(){return d})),t.d(n,"c",(function(){return b})),t.d(n,"e",(function(){return w}));var i=t(2);t.d(n,"d",(function(){return i.a})),t.d(n,"f",(function(){return i.b})),t.d(n,"g",(function(){return i.c})),t.d(n,"h",(function(){return i.d})),t.d(n,"i",(function(){return i.e})),t.d(n,"j",(function(){return i.f})),t.d(n,"k",(function(){return i.g})),t.d(n,"l",(function(){return i.i})),t.d(n,"m",(function(){return i.j})),t.d(n,"n",(function(){return i.k})),t.d(n,"o",(function(){return i.l})),t.d(n,"p",(function(){return i.m})),t.d(n,"q",(function(){return i.n})),t.d(n,"r",(function(){return i.o}));var r=t(4),o=t(0),a=t.n(o),l=t(6),s=(t(1),t(3)),c=t(7),u=t(5),p=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(l.a)(n.props),n}return Object(r.a)(n,e),n.prototype.render=function(){return a.a.createElement(i.e,{history:this.history,children:this.props.children})},n}(a.a.Component);var d=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(l.b)(n.props),n}return Object(r.a)(n,e),n.prototype.render=function(){return a.a.createElement(i.e,{history:this.history,children:this.props.children})},n}(a.a.Component);var f=function(e,n){return"function"==typeof e?e(n):e},m=function(e,n){return"string"==typeof e?Object(l.c)(e,null,null,n):e},g=function(e){return e},h=a.a.forwardRef;void 0===h&&(h=g);var v=h((function(e,n){var t=e.innerRef,i=e.navigate,r=e.onClick,o=Object(c.a)(e,["innerRef","navigate","onClick"]),l=o.target,u=Object(s.a)({},o,{onClick:function(e){try{r&&r(e)}catch(n){throw e.preventDefault(),n}e.defaultPrevented||0!==e.button||l&&"_self"!==l||function(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}(e)||(e.preventDefault(),i())}});return u.ref=g!==h&&n||t,a.a.createElement("a",u)}));var b=h((function(e,n){var t=e.component,r=void 0===t?v:t,o=e.replace,l=e.to,p=e.innerRef,d=Object(c.a)(e,["component","replace","to","innerRef"]);return a.a.createElement(i.h.Consumer,null,(function(e){e||Object(u.a)(!1);var t=e.history,i=m(f(l,e.location),e.location),c=i?t.createHref(i):"",v=Object(s.a)({},d,{href:c,navigate:function(){var n=f(l,e.location);(o?t.replace:t.push)(n)}});return g!==h?v.ref=n||p:v.innerRef=p,a.a.createElement(r,v)}))})),y=function(e){return e},k=a.a.forwardRef;void 0===k&&(k=y);var w=k((function(e,n){var t=e["aria-current"],r=void 0===t?"page":t,o=e.activeClassName,l=void 0===o?"active":o,p=e.activeStyle,d=e.className,g=e.exact,h=e.isActive,v=e.location,w=e.sensitive,_=e.strict,x=e.style,E=e.to,S=e.innerRef,T=Object(c.a)(e,["aria-current","activeClassName","activeStyle","className","exact","isActive","location","sensitive","strict","style","to","innerRef"]);return a.a.createElement(i.h.Consumer,null,(function(e){e||Object(u.a)(!1);var t=v||e.location,o=m(f(E,t),t),c=o.pathname,P=c&&c.replace(/([.+*?=^!:${}()[\]|/\\])/g,"\\$1"),C=P?Object(i.j)(t.pathname,{path:P,exact:g,sensitive:w,strict:_}):null,O=!!(h?h(C,t):C),A=O?function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];return n.filter((function(e){return e})).join(" ")}(d,l):d,R=O?Object(s.a)({},x,{},p):x,N=Object(s.a)({"aria-current":O&&r||null,className:A,style:R,to:o},T);return y!==k?N.ref=n||S:N.innerRef=S,a.a.createElement(b,N)}))}))},function(e,n,t){"use strict";t.r(n),n.default={title:"Netsage Pipeline Documentation",tagline:"Netsage Pipeline Documentation",url:"https://netsage-project.github.io",baseUrl:"/netsage-pipeline/",favicon:"img/favicon.ico",organizationName:"netsage-project",projectName:"netsage-pipeline",themeConfig:{navbar:{title:"NetSage Documentation",logo:{alt:"NetSage Logo",src:"img/logo.png"},items:[{label:"Pipeline Documentation",position:"left",activeBasePath:"docs",to:"docs/pipeline/intro"},{href:"https://netsage-project.github.io/netsage-grafana-configs/",label:"Dashboard Documentation",position:"left",target:"_self"},{type:"docsVersionDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{href:"https://github.com/netsage-project/netsage-pipeline/",label:"GitHub",position:"right"}],hideOnScroll:!1},footer:{style:"dark",links:[],copyright:"Copyright \xa9 2024 NetSage Built with Docusaurus."},colorMode:{defaultMode:"light",disableSwitch:!1,respectPrefersColorScheme:!1,switchConfig:{darkIcon:"\ud83c\udf1c",darkIconStyle:{},lightIcon:"\ud83c\udf1e",lightIconStyle:{}}},docs:{versionPersistence:"localStorage"},metadatas:[],prism:{additionalLanguages:[]},hideableSidebar:!1},presets:[["@docusaurus/preset-classic",{docs:{remarkPlugins:[null],sidebarPath:"/home/dojosout/netsage-pipeline/website/sidebars.js",lastVersion:"2.1.0",versions:{current:{label:"master (unreleased)"}},editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/"},blog:{showReadingTime:!0,editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/blog/"},theme:{customCss:"/home/dojosout/netsage-pipeline/website/src/css/custom.css"}}]],baseUrlIssueBanner:!0,i18n:{defaultLocale:"en",locales:["en"],localeConfigs:{}},onBrokenLinks:"throw",onBrokenMarkdownLinks:"warn",onDuplicateRoutes:"warn",customFields:{},plugins:[],themes:[],titleDelimiter:"|",noIndex:!1}},function(e,n,t){"use strict";var i=Object.getOwnPropertySymbols,r=Object.prototype.hasOwnProperty,o=Object.prototype.propertyIsEnumerable;function a(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var n={},t=0;t<10;t++)n["_"+String.fromCharCode(t)]=t;if("0123456789"!==Object.getOwnPropertyNames(n).map((function(e){return n[e]})).join(""))return!1;var i={};return"abcdefghijklmnopqrst".split("").forEach((function(e){i[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},i)).join("")}catch(r){return!1}}()?Object.assign:function(e,n){for(var t,l,s=a(e),c=1;c<arguments.length;c++){for(var u in t=Object(arguments[c]))r.call(t,u)&&(s[u]=t[u]);if(i){l=i(t);for(var p=0;p<l.length;p++)o.call(t,l[p])&&(s[l[p]]=t[l[p]])}}return s}},function(e){e.exports=JSON.parse('{"/netsage-pipeline/-421":{"component":"c4f5d8e4","config":"5e9f5e1a"},"/netsage-pipeline/docs/1.2.10-296":{"component":"1be78505","versionMetadata":"d5087a67"},"/netsage-pipeline/docs/1.2.10/components/docker_env-63f":{"component":"17896441","content":"e10970d8"},"/netsage-pipeline/docs/1.2.10/components/docker_first_steps-059":{"component":"17896441","content":"ae8a1364"},"/netsage-pipeline/docs/1.2.10/components/docker_pipeline-a0b":{"component":"17896441","content":"46a600e4"},"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install-ec8":{"component":"17896441","content":"e7670743"},"/netsage-pipeline/docs/1.2.10/deploy/choose_install-fb2":{"component":"17896441","content":"8bbfcfa9"},"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced-a83":{"component":"17896441","content":"f1ddb0ed"},"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple-15f":{"component":"17896441","content":"e746408e"},"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot-e2e":{"component":"17896441","content":"cd902920"},"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade-b01":{"component":"17896441","content":"38962575"},"/netsage-pipeline/docs/1.2.10/devel/dev_dataset-7fa":{"component":"17896441","content":"f485eca7"},"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide-f40":{"component":"17896441","content":"63f5fe43"},"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag-b8f":{"component":"17896441","content":"b83a1049"},"/netsage-pipeline/docs/1.2.10/devel/docusaurus-bcf":{"component":"17896441","content":"514d766a"},"/netsage-pipeline/docs/1.2.10/pipeline/elastic-c07":{"component":"17896441","content":"4a74d8e8"},"/netsage-pipeline/docs/1.2.10/pipeline/importer-3c0":{"component":"17896441","content":"799b1578"},"/netsage-pipeline/docs/1.2.10/pipeline/intro-3b3":{"component":"17896441","content":"49e74cdd"},"/netsage-pipeline/docs/1.2.10/pipeline/logstash-116":{"component":"17896441","content":"0271802f"},"/netsage-pipeline/docs/1.2.10/pipeline/nfdump-f4d":{"component":"17896441","content":"f8bc0717"},"/netsage-pipeline/docs/1.2.10/pipeline/tstat-4c2":{"component":"17896441","content":"0b8e461a"},"/netsage-pipeline/docs/1.2.11-d04":{"component":"1be78505","versionMetadata":"02341ef2"},"/netsage-pipeline/docs/1.2.11/components/docker_env-1d6":{"component":"17896441","content":"8997819f"},"/netsage-pipeline/docs/1.2.11/components/docker_first_steps-62e":{"component":"17896441","content":"e987ec87"},"/netsage-pipeline/docs/1.2.11/components/docker_pipeline-6b1":{"component":"17896441","content":"69c1008e"},"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install-c46":{"component":"17896441","content":"0327c8bf"},"/netsage-pipeline/docs/1.2.11/deploy/choose_install-f72":{"component":"17896441","content":"05c6fd0e"},"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced-0a8":{"component":"17896441","content":"22d86c11"},"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple-c8a":{"component":"17896441","content":"dd1a9f26"},"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot-a22":{"component":"17896441","content":"7c78db09"},"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade-9ae":{"component":"17896441","content":"d48b255e"},"/netsage-pipeline/docs/1.2.11/devel/dev_dataset-ba4":{"component":"17896441","content":"9a5c9747"},"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide-80d":{"component":"17896441","content":"41980e0e"},"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag-681":{"component":"17896441","content":"6d913b5d"},"/netsage-pipeline/docs/1.2.11/devel/docusaurus-e33":{"component":"17896441","content":"9d5eac8e"},"/netsage-pipeline/docs/1.2.11/pipeline/elastic-ef5":{"component":"17896441","content":"5b6c4cb7"},"/netsage-pipeline/docs/1.2.11/pipeline/importer-5cc":{"component":"17896441","content":"96852331"},"/netsage-pipeline/docs/1.2.11/pipeline/intro-5fd":{"component":"17896441","content":"a12215ea"},"/netsage-pipeline/docs/1.2.11/pipeline/logstash-0d3":{"component":"17896441","content":"012f72bc"},"/netsage-pipeline/docs/1.2.11/pipeline/nfdump-9b4":{"component":"17896441","content":"e30bbbcf"},"/netsage-pipeline/docs/1.2.11/pipeline/tstat-3cb":{"component":"17896441","content":"f52ba0ac"},"/netsage-pipeline/docs/1.2.5-c52":{"component":"1be78505","versionMetadata":"6c279c12"},"/netsage-pipeline/docs/1.2.5/deploy/choose_install-3ce":{"component":"17896441","content":"3179f6dc"},"/netsage-pipeline/docs/1.2.5/deploy/docker_install-d98":{"component":"17896441","content":"a04bc459"},"/netsage-pipeline/docs/1.2.5/deploy/install-47c":{"component":"17896441","content":"9fe55709"},"/netsage-pipeline/docs/1.2.5/devel/docker-0e4":{"component":"17896441","content":"83dbc9a5"},"/netsage-pipeline/docs/1.2.5/docusaurus-882":{"component":"17896441","content":"480c5911"},"/netsage-pipeline/docs/1.2.5/pipeline-8ff":{"component":"17896441","content":"c0bd86e8"},"/netsage-pipeline/docs/1.2.5/pipeline_importer-93b":{"component":"17896441","content":"066f8364"},"/netsage-pipeline/docs/1.2.5/pipeline_logstash-9ed":{"component":"17896441","content":"a7459da2"},"/netsage-pipeline/docs/1.2.6-489":{"component":"1be78505","versionMetadata":"7d380c4c"},"/netsage-pipeline/docs/1.2.6/deploy/choose_install-0d1":{"component":"17896441","content":"d5479f42"},"/netsage-pipeline/docs/1.2.6/deploy/docker_install-4df":{"component":"17896441","content":"9feaf8ab"},"/netsage-pipeline/docs/1.2.6/deploy/install-8ee":{"component":"17896441","content":"e2be698d"},"/netsage-pipeline/docs/1.2.6/devel/docker-d95":{"component":"17896441","content":"aa98ec4a"},"/netsage-pipeline/docs/1.2.6/docusaurus-312":{"component":"17896441","content":"d91f9def"},"/netsage-pipeline/docs/1.2.6/pipeline-7ea":{"component":"17896441","content":"ebfaece6"},"/netsage-pipeline/docs/1.2.6/pipeline_importer-c76":{"component":"17896441","content":"21ee0bc4"},"/netsage-pipeline/docs/1.2.6/pipeline_logstash-ee6":{"component":"17896441","content":"38d1fe39"},"/netsage-pipeline/docs/1.2.7-3e0":{"component":"1be78505","versionMetadata":"906738b8"},"/netsage-pipeline/docs/1.2.7/components/docker_env-fb9":{"component":"17896441","content":"358e6759"},"/netsage-pipeline/docs/1.2.7/components/docker_pipeline-28e":{"component":"17896441","content":"e436ce66"},"/netsage-pipeline/docs/1.2.7/components/docker_upgrade-aca":{"component":"17896441","content":"4ff24bf8"},"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install-88b":{"component":"17896441","content":"aafd28b2"},"/netsage-pipeline/docs/1.2.7/deploy/choose_install-9c2":{"component":"17896441","content":"5c4beb71"},"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced-a68":{"component":"17896441","content":"ac3b7d69"},"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple-c15":{"component":"17896441","content":"879923b2"},"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot-f5e":{"component":"17896441","content":"d71ff009"},"/netsage-pipeline/docs/1.2.7/devel/dev_dataset-c2f":{"component":"17896441","content":"3914abfa"},"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide-13c":{"component":"17896441","content":"97edd56e"},"/netsage-pipeline/docs/1.2.7/docusaurus-fba":{"component":"17896441","content":"0381afdd"},"/netsage-pipeline/docs/1.2.7/pipeline/elastic-545":{"component":"17896441","content":"3309db24"},"/netsage-pipeline/docs/1.2.7/pipeline/importer-457":{"component":"17896441","content":"43862f47"},"/netsage-pipeline/docs/1.2.7/pipeline/intro-d5a":{"component":"17896441","content":"07faddd5"},"/netsage-pipeline/docs/1.2.7/pipeline/logstash-144":{"component":"17896441","content":"bb237b1e"},"/netsage-pipeline/docs/1.2.7/pipeline/nfdump-6d8":{"component":"17896441","content":"42f7ba93"},"/netsage-pipeline/docs/1.2.7/pipeline/tstat-d6c":{"component":"17896441","content":"e186625e"},"/netsage-pipeline/docs/1.2.8-51d":{"component":"1be78505","versionMetadata":"992819d4"},"/netsage-pipeline/docs/1.2.8/components/docker_env-f3f":{"component":"17896441","content":"abbf3a21"},"/netsage-pipeline/docs/1.2.8/components/docker_first_steps-d25":{"component":"17896441","content":"9bfdba8c"},"/netsage-pipeline/docs/1.2.8/components/docker_pipeline-7d7":{"component":"17896441","content":"42f0e773"},"/netsage-pipeline/docs/1.2.8/components/docker_upgrade-db5":{"component":"17896441","content":"a55d7fc8"},"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install-a0d":{"component":"17896441","content":"49703371"},"/netsage-pipeline/docs/1.2.8/deploy/choose_install-298":{"component":"17896441","content":"d2c4f39e"},"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced-150":{"component":"17896441","content":"56776a50"},"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple-f57":{"component":"17896441","content":"42e094bc"},"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot-a18":{"component":"17896441","content":"d5140579"},"/netsage-pipeline/docs/1.2.8/devel/dev_dataset-744":{"component":"17896441","content":"e00e17a0"},"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide-e12":{"component":"17896441","content":"0214960a"},"/netsage-pipeline/docs/1.2.8/docusaurus-57f":{"component":"17896441","content":"58d958f4"},"/netsage-pipeline/docs/1.2.8/pipeline/elastic-e1a":{"component":"17896441","content":"bed7ab4e"},"/netsage-pipeline/docs/1.2.8/pipeline/importer-b38":{"component":"17896441","content":"ad308aff"},"/netsage-pipeline/docs/1.2.8/pipeline/intro-c0b":{"component":"17896441","content":"33f6b1c9"},"/netsage-pipeline/docs/1.2.8/pipeline/logstash-00d":{"component":"17896441","content":"6a059f5b"},"/netsage-pipeline/docs/1.2.8/pipeline/nfdump-6f1":{"component":"17896441","content":"b1bfe9f5"},"/netsage-pipeline/docs/1.2.8/pipeline/tstat-59a":{"component":"17896441","content":"832ca04e"},"/netsage-pipeline/docs/1.2.9-cb5":{"component":"1be78505","versionMetadata":"d75898ab"},"/netsage-pipeline/docs/1.2.9/components/docker_env-e31":{"component":"17896441","content":"6125e2e5"},"/netsage-pipeline/docs/1.2.9/components/docker_first_steps-253":{"component":"17896441","content":"b3c114f4"},"/netsage-pipeline/docs/1.2.9/components/docker_pipeline-6e2":{"component":"17896441","content":"d1597237"},"/netsage-pipeline/docs/1.2.9/components/docker_upgrade-8be":{"component":"17896441","content":"6546ef57"},"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install-31c":{"component":"17896441","content":"56a29d1a"},"/netsage-pipeline/docs/1.2.9/deploy/choose_install-5ff":{"component":"17896441","content":"f64b3473"},"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced-711":{"component":"17896441","content":"5ee17acf"},"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple-536":{"component":"17896441","content":"c51f4c72"},"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot-7d0":{"component":"17896441","content":"1d038b8a"},"/netsage-pipeline/docs/1.2.9/devel/dev_dataset-e4a":{"component":"17896441","content":"40b960a4"},"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide-b39":{"component":"17896441","content":"9b505a1f"},"/netsage-pipeline/docs/1.2.9/docusaurus-bed":{"component":"17896441","content":"db0ed613"},"/netsage-pipeline/docs/1.2.9/pipeline/elastic-bd5":{"component":"17896441","content":"5a20f281"},"/netsage-pipeline/docs/1.2.9/pipeline/importer-90f":{"component":"17896441","content":"99d6017e"},"/netsage-pipeline/docs/1.2.9/pipeline/intro-e91":{"component":"17896441","content":"c3f49368"},"/netsage-pipeline/docs/1.2.9/pipeline/logstash-9e9":{"component":"17896441","content":"9720d299"},"/netsage-pipeline/docs/1.2.9/pipeline/nfdump-220":{"component":"17896441","content":"7b85b03c"},"/netsage-pipeline/docs/1.2.9/pipeline/tstat-341":{"component":"17896441","content":"50032990"},"/netsage-pipeline/docs/next-47f":{"component":"1be78505","versionMetadata":"935f2afb"},"/netsage-pipeline/docs/next/components/docker_env-cfe":{"component":"17896441","content":"64269ed8"},"/netsage-pipeline/docs/next/components/docker_first_steps-e72":{"component":"17896441","content":"080a21b4"},"/netsage-pipeline/docs/next/components/docker_pipeline-5eb":{"component":"17896441","content":"ba55e963"},"/netsage-pipeline/docs/next/deploy/docker_install_advanced-de0":{"component":"17896441","content":"cbd1c0d0"},"/netsage-pipeline/docs/next/deploy/docker_install_simple-72d":{"component":"17896441","content":"734dc415"},"/netsage-pipeline/docs/next/deploy/docker_troubleshoot-593":{"component":"17896441","content":"653602f1"},"/netsage-pipeline/docs/next/deploy/docker_upgrade-68f":{"component":"17896441","content":"ceb45c47"},"/netsage-pipeline/docs/next/devel/dev_dataset-091":{"component":"17896441","content":"fa16f0fc"},"/netsage-pipeline/docs/next/devel/docker_dev_guide-85f":{"component":"17896441","content":"1f3a3353"},"/netsage-pipeline/docs/next/devel/docker_dev_tag-e93":{"component":"17896441","content":"16fcefd4"},"/netsage-pipeline/docs/next/devel/docusaurus-702":{"component":"17896441","content":"4ffdf56a"},"/netsage-pipeline/docs/next/pipeline/elastic-533":{"component":"17896441","content":"7d0b7755"},"/netsage-pipeline/docs/next/pipeline/importer-063":{"component":"17896441","content":"d3556f45"},"/netsage-pipeline/docs/next/pipeline/intro-f04":{"component":"17896441","content":"22b7278f"},"/netsage-pipeline/docs/next/pipeline/logstash-aec":{"component":"17896441","content":"bd6c6f88"},"/netsage-pipeline/docs/next/pipeline/nfdump-543":{"component":"17896441","content":"1cc43674"},"/netsage-pipeline/docs/next/pipeline/tstat-474":{"component":"17896441","content":"e2c161d9"},"/netsage-pipeline/docs-0a0":{"component":"1be78505","versionMetadata":"63503174"},"/netsage-pipeline/docs/components/docker_env-a6d":{"component":"17896441","content":"8b301107"},"/netsage-pipeline/docs/components/docker_first_steps-fc9":{"component":"17896441","content":"ff1d9e71"},"/netsage-pipeline/docs/components/docker_pipeline-40b":{"component":"17896441","content":"6e2a1c72"},"/netsage-pipeline/docs/deploy/docker_install_advanced-de0":{"component":"17896441","content":"9f628d52"},"/netsage-pipeline/docs/deploy/docker_install_simple-6e9":{"component":"17896441","content":"12ba72b5"},"/netsage-pipeline/docs/deploy/docker_troubleshoot-8df":{"component":"17896441","content":"86d09e6e"},"/netsage-pipeline/docs/deploy/docker_upgrade-bcf":{"component":"17896441","content":"9191c864"},"/netsage-pipeline/docs/devel/dev_dataset-e5f":{"component":"17896441","content":"12ecfc4a"},"/netsage-pipeline/docs/devel/docker_dev_guide-81e":{"component":"17896441","content":"0dd3dba6"},"/netsage-pipeline/docs/devel/docker_dev_tag-8f8":{"component":"17896441","content":"6e3166bc"},"/netsage-pipeline/docs/devel/docusaurus-78c":{"component":"17896441","content":"d74c151a"},"/netsage-pipeline/docs/pipeline/elastic-d63":{"component":"17896441","content":"71e5fd50"},"/netsage-pipeline/docs/pipeline/importer-e01":{"component":"17896441","content":"5ef24f1d"},"/netsage-pipeline/docs/pipeline/intro-85d":{"component":"17896441","content":"f2cce206"},"/netsage-pipeline/docs/pipeline/logstash-bf4":{"component":"17896441","content":"f0b0282f"},"/netsage-pipeline/docs/pipeline/nfdump-adf":{"component":"17896441","content":"092c2d1d"},"/netsage-pipeline/docs/pipeline/tstat-9b0":{"component":"17896441","content":"f9995f72"}}')},function(e,n,t){"use strict";var i=t(0),r=t.n(i);n.a=r.a.createContext(null)},function(e,n,t){var i,r;void 0===(r="function"==typeof(i=function(){var e,n,t={version:"0.2.0"},i=t.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'<div class="bar" role="bar"><div class="peg"></div></div><div class="spinner" role="spinner"><div class="spinner-icon"></div></div>'};function r(e,n,t){return e<n?n:e>t?t:e}function o(e){return 100*(-1+e)}function a(e,n,t){var r;return(r="translate3d"===i.positionUsing?{transform:"translate3d("+o(e)+"%,0,0)"}:"translate"===i.positionUsing?{transform:"translate("+o(e)+"%,0)"}:{"margin-left":o(e)+"%"}).transition="all "+n+"ms "+t,r}t.configure=function(e){var n,t;for(n in e)void 0!==(t=e[n])&&e.hasOwnProperty(n)&&(i[n]=t);return this},t.status=null,t.set=function(e){var n=t.isStarted();e=r(e,i.minimum,1),t.status=1===e?null:e;var o=t.render(!n),c=o.querySelector(i.barSelector),u=i.speed,p=i.easing;return o.offsetWidth,l((function(n){""===i.positionUsing&&(i.positionUsing=t.getPositioningCSS()),s(c,a(e,u,p)),1===e?(s(o,{transition:"none",opacity:1}),o.offsetWidth,setTimeout((function(){s(o,{transition:"all "+u+"ms linear",opacity:0}),setTimeout((function(){t.remove(),n()}),u)}),u)):setTimeout(n,u)})),this},t.isStarted=function(){return"number"==typeof t.status},t.start=function(){t.status||t.set(0);var e=function(){setTimeout((function(){t.status&&(t.trickle(),e())}),i.trickleSpeed)};return i.trickle&&e(),this},t.done=function(e){return e||t.status?t.inc(.3+.5*Math.random()).set(1):this},t.inc=function(e){var n=t.status;return n?("number"!=typeof e&&(e=(1-n)*r(Math.random()*n,.1,.95)),n=r(n+e,0,.994),t.set(n)):t.start()},t.trickle=function(){return t.inc(Math.random()*i.trickleRate)},e=0,n=0,t.promise=function(i){return i&&"resolved"!==i.state()?(0===n&&t.start(),e++,n++,i.always((function(){0==--n?(e=0,t.done()):t.set((e-n)/e)})),this):this},t.render=function(e){if(t.isRendered())return document.getElementById("nprogress");u(document.documentElement,"nprogress-busy");var n=document.createElement("div");n.id="nprogress",n.innerHTML=i.template;var r,a=n.querySelector(i.barSelector),l=e?"-100":o(t.status||0),c=document.querySelector(i.parent);return s(a,{transition:"all 0 linear",transform:"translate3d("+l+"%,0,0)"}),i.showSpinner||(r=n.querySelector(i.spinnerSelector))&&f(r),c!=document.body&&u(c,"nprogress-custom-parent"),c.appendChild(n),n},t.remove=function(){p(document.documentElement,"nprogress-busy"),p(document.querySelector(i.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&f(e)},t.isRendered=function(){return!!document.getElementById("nprogress")},t.getPositioningCSS=function(){var e=document.body.style,n="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return n+"Perspective"in e?"translate3d":n+"Transform"in e?"translate":"margin"};var l=function(){var e=[];function n(){var t=e.shift();t&&t(n)}return function(t){e.push(t),1==e.length&&n()}}(),s=function(){var e=["Webkit","O","Moz","ms"],n={};function t(e){return e.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,(function(e,n){return n.toUpperCase()}))}function i(n){var t=document.body.style;if(n in t)return n;for(var i,r=e.length,o=n.charAt(0).toUpperCase()+n.slice(1);r--;)if((i=e[r]+o)in t)return i;return n}function r(e){return e=t(e),n[e]||(n[e]=i(e))}function o(e,n,t){n=r(n),e.style[n]=t}return function(e,n){var t,i,r=arguments;if(2==r.length)for(t in n)void 0!==(i=n[t])&&n.hasOwnProperty(t)&&o(e,t,i);else o(e,r[1],r[2])}}();function c(e,n){return("string"==typeof e?e:d(e)).indexOf(" "+n+" ")>=0}function u(e,n){var t=d(e),i=t+n;c(t,n)||(e.className=i.substring(1))}function p(e,n){var t,i=d(e);c(e,n)&&(t=i.replace(" "+n+" "," "),e.className=t.substring(1,t.length-1))}function d(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function f(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return t})?i.call(n,t,n,e):i)||(e.exports=r)},function(e,n,t){"use strict";t.r(n);var i=t(0),r=t(14);n.default=function(){var e=Object(i.useContext)(r.a);if(null===e)throw new Error("Docusaurus context not provided");return e}},,,function(e,n,t){var i={"./":48};function r(e){var n=o(e);return t(n)}function o(e){if(!t.o(i,e)){var n=new Error("Cannot find module '"+e+"'");throw n.code="MODULE_NOT_FOUND",n}return i[e]}r.keys=function(){return Object.keys(i)},r.resolve=o,e.exports=r,r.id=19},function(e,n,t){"use strict";function i(e,n){e.prototype=Object.create(n.prototype),e.prototype.constructor=e,e.__proto__=n}function r(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(){return(a=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e}).apply(this,arguments)}var l=t(0),s=t(1),c=[],u=[];function p(e){var n=e(),t={loading:!0,loaded:null,error:null};return t.promise=n.then((function(e){return t.loading=!1,t.loaded=e,e})).catch((function(e){throw t.loading=!1,t.error=e,e})),t}function d(e){var n={loading:!1,loaded:{},error:null},t=[];try{Object.keys(e).forEach((function(i){var r=p(e[i]);r.loading?n.loading=!0:(n.loaded[i]=r.loaded,n.error=r.error),t.push(r.promise),r.promise.then((function(e){n.loaded[i]=e})).catch((function(e){n.error=e}))}))}catch(i){n.error=i}return n.promise=Promise.all(t).then((function(e){return n.loading=!1,e})).catch((function(e){throw n.loading=!1,e})),n}function f(e,n){return l.createElement((t=e)&&t.__esModule?t.default:t,n);var t}function m(e,n){var p,d;if(!n.loading)throw new Error("react-loadable requires a `loading` component");var m=a({loader:null,loading:null,delay:200,timeout:null,render:f,webpack:null,modules:null},n),g=null;function h(){return g||(g=e(m.loader)),g.promise}return c.push(h),"function"==typeof m.webpack&&u.push((function(){if((0,m.webpack)().every((function(e){return void 0!==e&&void 0!==t.m[e]})))return h()})),d=p=function(n){function t(t){var i;return o(r(r(i=n.call(this,t)||this)),"retry",(function(){i.setState({error:null,loading:!0,timedOut:!1}),g=e(m.loader),i._loadModule()})),h(),i.state={error:g.error,pastDelay:!1,timedOut:!1,loading:g.loading,loaded:g.loaded},i}i(t,n),t.preload=function(){return h()};var a=t.prototype;return a.UNSAFE_componentWillMount=function(){this._loadModule()},a.componentDidMount=function(){this._mounted=!0},a._loadModule=function(){var e=this;if(this.context.loadable&&Array.isArray(m.modules)&&m.modules.forEach((function(n){e.context.loadable.report(n)})),g.loading){var n=function(n){e._mounted&&e.setState(n)};"number"==typeof m.delay&&(0===m.delay?this.setState({pastDelay:!0}):this._delay=setTimeout((function(){n({pastDelay:!0})}),m.delay)),"number"==typeof m.timeout&&(this._timeout=setTimeout((function(){n({timedOut:!0})}),m.timeout));var t=function(){n({error:g.error,loaded:g.loaded,loading:g.loading}),e._clearTimeouts()};g.promise.then((function(){return t(),null})).catch((function(e){return t(),null}))}},a.componentWillUnmount=function(){this._mounted=!1,this._clearTimeouts()},a._clearTimeouts=function(){clearTimeout(this._delay),clearTimeout(this._timeout)},a.render=function(){return this.state.loading||this.state.error?l.createElement(m.loading,{isLoading:this.state.loading,pastDelay:this.state.pastDelay,timedOut:this.state.timedOut,error:this.state.error,retry:this.retry}):this.state.loaded?m.render(this.state.loaded,this.props):null},t}(l.Component),o(p,"contextTypes",{loadable:s.shape({report:s.func.isRequired})}),d}function g(e){return m(p,e)}g.Map=function(e){if("function"!=typeof e.render)throw new Error("LoadableMap requires a `render(loaded, props)` function");return m(d,e)};var h=function(e){function n(){return e.apply(this,arguments)||this}i(n,e);var t=n.prototype;return t.getChildContext=function(){return{loadable:{report:this.props.report}}},t.render=function(){return l.Children.only(this.props.children)},n}(l.Component);function v(e){for(var n=[];e.length;){var t=e.pop();n.push(t())}return Promise.all(n).then((function(){if(e.length)return v(e)}))}o(h,"propTypes",{report:s.func.isRequired}),o(h,"childContextTypes",{loadable:s.shape({report:s.func.isRequired}).isRequired}),g.Capture=h,g.preloadAll=function(){return new Promise((function(e,n){v(c).then(e,n)}))},g.preloadReady=function(){return new Promise((function(e,n){v(u).then(e,e)}))},e.exports=g},function(e,n,t){"use strict";(function(e){var i=t(0),r=t.n(i),o=t(4),a=t(1),l=t.n(a),s=1073741823,c="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:void 0!==e?e:{};function u(e){var n=[];return{on:function(e){n.push(e)},off:function(e){n=n.filter((function(n){return n!==e}))},get:function(){return e},set:function(t,i){e=t,n.forEach((function(n){return n(e,i)}))}}}var p=r.a.createContext||function(e,n){var t,r,a,p="__create-react-context-"+((c[a="__global_unique_id__"]=(c[a]||0)+1)+"__"),d=function(e){function t(){var n;return(n=e.apply(this,arguments)||this).emitter=u(n.props.value),n}Object(o.a)(t,e);var i=t.prototype;return i.getChildContext=function(){var e;return(e={})[p]=this.emitter,e},i.componentWillReceiveProps=function(e){if(this.props.value!==e.value){var t,i=this.props.value,r=e.value;((o=i)===(a=r)?0!==o||1/o==1/a:o!=o&&a!=a)?t=0:(t="function"==typeof n?n(i,r):s,0!==(t|=0)&&this.emitter.set(e.value,t))}var o,a},i.render=function(){return this.props.children},t}(i.Component);d.childContextTypes=((t={})[p]=l.a.object.isRequired,t);var f=function(n){function t(){var e;return(e=n.apply(this,arguments)||this).state={value:e.getValue()},e.onUpdate=function(n,t){0!=((0|e.observedBits)&t)&&e.setState({value:e.getValue()})},e}Object(o.a)(t,n);var i=t.prototype;return i.componentWillReceiveProps=function(e){var n=e.observedBits;this.observedBits=null==n?s:n},i.componentDidMount=function(){this.context[p]&&this.context[p].on(this.onUpdate);var e=this.props.observedBits;this.observedBits=null==e?s:e},i.componentWillUnmount=function(){this.context[p]&&this.context[p].off(this.onUpdate)},i.getValue=function(){return this.context[p]?this.context[p].get():e},i.render=function(){return(e=this.props.children,Array.isArray(e)?e[0]:e)(this.state.value);var e},t}(i.Component);return f.contextTypes=((r={})[p]=l.a.object,r),{Provider:d,Consumer:f}};n.a=p}).call(this,t(28))},function(e,n,t){var i=t(45);e.exports=f,e.exports.parse=o,e.exports.compile=function(e,n){return l(o(e,n),n)},e.exports.tokensToFunction=l,e.exports.tokensToRegExp=d;var r=new RegExp(["(\\\\.)","([\\/.])?(?:(?:\\:(\\w+)(?:\\(((?:\\\\.|[^\\\\()])+)\\))?|\\(((?:\\\\.|[^\\\\()])+)\\))([+*?])?|(\\*))"].join("|"),"g");function o(e,n){for(var t,i=[],o=0,a=0,l="",u=n&&n.delimiter||"/";null!=(t=r.exec(e));){var p=t[0],d=t[1],f=t.index;if(l+=e.slice(a,f),a=f+p.length,d)l+=d[1];else{var m=e[a],g=t[2],h=t[3],v=t[4],b=t[5],y=t[6],k=t[7];l&&(i.push(l),l="");var w=null!=g&&null!=m&&m!==g,_="+"===y||"*"===y,x="?"===y||"*"===y,E=t[2]||u,S=v||b;i.push({name:h||o++,prefix:g||"",delimiter:E,optional:x,repeat:_,partial:w,asterisk:!!k,pattern:S?c(S):k?".*":"[^"+s(E)+"]+?"})}}return a<e.length&&(l+=e.substr(a)),l&&i.push(l),i}function a(e){return encodeURI(e).replace(/[\/?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()}))}function l(e,n){for(var t=new Array(e.length),r=0;r<e.length;r++)"object"==typeof e[r]&&(t[r]=new RegExp("^(?:"+e[r].pattern+")$",p(n)));return function(n,r){for(var o="",l=n||{},s=(r||{}).pretty?a:encodeURIComponent,c=0;c<e.length;c++){var u=e[c];if("string"!=typeof u){var p,d=l[u.name];if(null==d){if(u.optional){u.partial&&(o+=u.prefix);continue}throw new TypeError('Expected "'+u.name+'" to be defined')}if(i(d)){if(!u.repeat)throw new TypeError('Expected "'+u.name+'" to not repeat, but received `'+JSON.stringify(d)+"`");if(0===d.length){if(u.optional)continue;throw new TypeError('Expected "'+u.name+'" to not be empty')}for(var f=0;f<d.length;f++){if(p=s(d[f]),!t[c].test(p))throw new TypeError('Expected all "'+u.name+'" to match "'+u.pattern+'", but received `'+JSON.stringify(p)+"`");o+=(0===f?u.prefix:u.delimiter)+p}}else{if(p=u.asterisk?encodeURI(d).replace(/[?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()})):s(d),!t[c].test(p))throw new TypeError('Expected "'+u.name+'" to match "'+u.pattern+'", but received "'+p+'"');o+=u.prefix+p}}else o+=u}return o}}function s(e){return e.replace(/([.+*?=^!:${}()[\]|\/\\])/g,"\\$1")}function c(e){return e.replace(/([=!:$\/()])/g,"\\$1")}function u(e,n){return e.keys=n,e}function p(e){return e&&e.sensitive?"":"i"}function d(e,n,t){i(n)||(t=n||t,n=[]);for(var r=(t=t||{}).strict,o=!1!==t.end,a="",l=0;l<e.length;l++){var c=e[l];if("string"==typeof c)a+=s(c);else{var d=s(c.prefix),f="(?:"+c.pattern+")";n.push(c),c.repeat&&(f+="(?:"+d+f+")*"),a+=f=c.optional?c.partial?d+"("+f+")?":"(?:"+d+"("+f+"))?":d+"("+f+")"}}var m=s(t.delimiter||"/"),g=a.slice(-m.length)===m;return r||(a=(g?a.slice(0,-m.length):a)+"(?:"+m+"(?=$))?"),a+=o?"$":r&&g?"":"(?="+m+"|$)",u(new RegExp("^"+a,p(t)),n)}function f(e,n,t){return i(n)||(t=n||t,n=[]),t=t||{},e instanceof RegExp?function(e,n){var t=e.source.match(/\((?!\?)/g);if(t)for(var i=0;i<t.length;i++)n.push({name:i,prefix:null,delimiter:null,optional:!1,repeat:!1,partial:!1,asterisk:!1,pattern:null});return u(e,n)}(e,n):i(e)?function(e,n,t){for(var i=[],r=0;r<e.length;r++)i.push(f(e[r],n,t).source);return u(new RegExp("(?:"+i.join("|")+")",p(t)),n)}(e,n,t):function(e,n,t){return d(o(e,t),n,t)}(e,n,t)}},function(e,n,t){"use strict";t.r(n);var i=t(10);t.d(n,"MemoryRouter",(function(){return i.d})),t.d(n,"Prompt",(function(){return i.f})),t.d(n,"Redirect",(function(){return i.g})),t.d(n,"Route",(function(){return i.h})),t.d(n,"Router",(function(){return i.i})),t.d(n,"StaticRouter",(function(){return i.j})),t.d(n,"Switch",(function(){return i.k})),t.d(n,"generatePath",(function(){return i.l})),t.d(n,"matchPath",(function(){return i.m})),t.d(n,"useHistory",(function(){return i.n})),t.d(n,"useLocation",(function(){return i.o})),t.d(n,"useParams",(function(){return i.p})),t.d(n,"useRouteMatch",(function(){return i.q})),t.d(n,"withRouter",(function(){return i.r})),t.d(n,"BrowserRouter",(function(){return i.a})),t.d(n,"HashRouter",(function(){return i.b})),t.d(n,"Link",(function(){return i.c})),t.d(n,"NavLink",(function(){return i.e}))},function(e,n,t){"use strict";var i,r,o,a=(i=0,r={util:{encode:function(e){return e instanceof o?new o(e.type,r.util.encode(e.content),e.alias):"Array"===r.util.type(e)?e.map(r.util.encode):e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).match(/\[object (\w+)\]/)[1]},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++i}),e.__id},clone:function(e,n){var t=r.util.type(e);switch(n=n||{},t){case"Object":if(n[r.util.objId(e)])return n[r.util.objId(e)];var i={};for(var o in n[r.util.objId(e)]=i,e)e.hasOwnProperty(o)&&(i[o]=r.util.clone(e[o],n));return i;case"Array":return n[r.util.objId(e)]?n[r.util.objId(e)]:(i=[],n[r.util.objId(e)]=i,e.forEach((function(e,t){i[t]=r.util.clone(e,n)})),i)}return e}},languages:{extend:function(e,n){var t=r.util.clone(r.languages[e]);for(var i in n)t[i]=n[i];return t},insertBefore:function(e,n,t,i){var o=(i=i||r.languages)[e];if(2==arguments.length){for(var a in t=arguments[1])t.hasOwnProperty(a)&&(o[a]=t[a]);return o}var l={};for(var s in o)if(o.hasOwnProperty(s)){if(s==n)for(var a in t)t.hasOwnProperty(a)&&(l[a]=t[a]);l[s]=o[s]}return r.languages.DFS(r.languages,(function(n,t){t===i[e]&&n!=e&&(this[n]=l)})),i[e]=l},DFS:function(e,n,t,i){for(var o in i=i||{},e)e.hasOwnProperty(o)&&(n.call(e,o,e[o],t||o),"Object"!==r.util.type(e[o])||i[r.util.objId(e[o])]?"Array"!==r.util.type(e[o])||i[r.util.objId(e[o])]||(i[r.util.objId(e[o])]=!0,r.languages.DFS(e[o],n,o,i)):(i[r.util.objId(e[o])]=!0,r.languages.DFS(e[o],n,null,i)))}},plugins:{},highlight:function(e,n,t){var i={code:e,grammar:n,language:t};return r.hooks.run("before-tokenize",i),i.tokens=r.tokenize(i.code,i.grammar),r.hooks.run("after-tokenize",i),o.stringify(r.util.encode(i.tokens),i.language)},matchGrammar:function(e,n,t,i,o,a,l){var s=r.Token;for(var c in t)if(t.hasOwnProperty(c)&&t[c]){if(c==l)return;var u=t[c];u="Array"===r.util.type(u)?u:[u];for(var p=0;p<u.length;++p){var d=u[p],f=d.inside,m=!!d.lookbehind,g=!!d.greedy,h=0,v=d.alias;if(g&&!d.pattern.global){var b=d.pattern.toString().match(/[imuy]*$/)[0];d.pattern=RegExp(d.pattern.source,b+"g")}d=d.pattern||d;for(var y=i,k=o;y<n.length;k+=n[y].length,++y){var w=n[y];if(n.length>e.length)return;if(!(w instanceof s)){if(g&&y!=n.length-1){if(d.lastIndex=k,!(P=d.exec(e)))break;for(var _=P.index+(m?P[1].length:0),x=P.index+P[0].length,E=y,S=k,T=n.length;E<T&&(S<x||!n[E].type&&!n[E-1].greedy);++E)_>=(S+=n[E].length)&&(++y,k=S);if(n[y]instanceof s)continue;C=E-y,w=e.slice(k,S),P.index-=k}else{d.lastIndex=0;var P=d.exec(w),C=1}if(P){m&&(h=P[1]?P[1].length:0),x=(_=P.index+h)+(P=P[0].slice(h)).length;var O=w.slice(0,_),A=w.slice(x),R=[y,C];O&&(++y,k+=O.length,R.push(O));var N=new s(c,f?r.tokenize(P,f):P,v,P,g);if(R.push(N),A&&R.push(A),Array.prototype.splice.apply(n,R),1!=C&&r.matchGrammar(e,n,t,y,k,!0,c),a)break}else if(a)break}}}}},hooks:{add:function(){},run:function(e,n){}},tokenize:function(e,n,t){var i=[e],o=n.rest;if(o){for(var a in o)n[a]=o[a];delete n.rest}return r.matchGrammar(e,i,n,0,0,!1),i}},(o=r.Token=function(e,n,t,i,r){this.type=e,this.content=n,this.alias=t,this.length=0|(i||"").length,this.greedy=!!r}).stringify=function(e,n,t){if("string"==typeof e)return e;if("Array"===r.util.type(e))return e.map((function(t){return o.stringify(t,n,e)})).join("");var i={type:e.type,content:o.stringify(e.content,n,t),tag:"span",classes:["token",e.type],attributes:{},language:n,parent:t};if(e.alias){var a="Array"===r.util.type(e.alias)?e.alias:[e.alias];Array.prototype.push.apply(i.classes,a)}var l=Object.keys(i.attributes).map((function(e){return e+'="'+(i.attributes[e]||"").replace(/"/g,"&quot;")+'"'})).join(" ");return"<"+i.tag+' class="'+i.classes.join(" ")+'"'+(l?" "+l:"")+">"+i.content+"</"+i.tag+">"},r);a.languages.markup={comment:/<!--[\s\S]*?-->/,prolog:/<\?[\s\S]+?\?>/,doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|<!--(?:[^-]|-(?!->))*-->)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^<!|>$|[[\]]/,"doctype-tag":/^DOCTYPE/,name:/[^\s<>'"]+/}},cdata:/<!\[CDATA\[[\s\S]*?]]>/i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},a.languages.markup.tag.inside["attr-value"].inside.entity=a.languages.markup.entity,a.languages.markup.doctype.inside["internal-subset"].inside=a.languages.markup,a.hooks.add("wrap",(function(e){"entity"===e.type&&(e.attributes.title=e.content.replace(/&amp;/,"&"))})),Object.defineProperty(a.languages.markup.tag,"addInlined",{value:function(e,n){var t={};t["language-"+n]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:a.languages[n]},t.cdata=/^<!\[CDATA\[|\]\]>$/i;var i={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:t}};i["language-"+n]={pattern:/[\s\S]+/,inside:a.languages[n]};var r={};r[e]={pattern:RegExp(/(<__[^>]*>)(?:<!\[CDATA\[(?:[^\]]|\](?!\]>))*\]\]>|(?!<!\[CDATA\[)[\s\S])*?(?=<\/__>)/.source.replace(/__/g,(function(){return e})),"i"),lookbehind:!0,greedy:!0,inside:i},a.languages.insertBefore("markup","cdata",r)}}),a.languages.html=a.languages.markup,a.languages.mathml=a.languages.markup,a.languages.svg=a.languages.markup,a.languages.xml=a.languages.extend("markup",{}),a.languages.ssml=a.languages.xml,a.languages.atom=a.languages.xml,a.languages.rss=a.languages.xml,function(e){var n="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",t={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},i={bash:t,environment:{pattern:RegExp("\\$"+n),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/=?|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+n),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|x[0-9a-fA-F]{1,2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)\w+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b\w+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+n),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+?)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:i},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:t}},{pattern:/(^|[^\\](?:\\\\)*)(["'])(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|(?!\2)[^\\`$])*\2/,lookbehind:!0,greedy:!0,inside:i}],environment:{pattern:RegExp("\\$?"+n),alias:"constant"},variable:i.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|aptitude|apt-cache|apt-get|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:if|then|else|elif|fi|for|while|in|case|esac|function|select|do|done|until)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|break|cd|continue|eval|exec|exit|export|getopts|hash|pwd|readonly|return|shift|test|times|trap|umask|unset|alias|bind|builtin|caller|command|declare|echo|enable|help|let|local|logout|mapfile|printf|read|readarray|source|type|typeset|ulimit|unalias|set|shopt)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:true|false)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|==?|!=?|=~|<<[<-]?|[&\d]?>>|\d?[<>]&?|&[>&]?|\|[&|]?|<=?|>=?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},t.inside=e.languages.bash;for(var r=["comment","function-name","for-or-select","assign-left","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],o=i.variable[1].inside,a=0;a<r.length;a++)o[r[a]]=e.languages.bash[r[a]];e.languages.shell=e.languages.bash}(a),a.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|interface|extends|implements|trait|instanceof|new)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue)\b/,boolean:/\b(?:true|false)\b/,function:/\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/},a.languages.c=a.languages.extend("clike",{comment:{pattern:/\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},"class-name":{pattern:/(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/,lookbehind:!0},keyword:/\b(?:__attribute__|_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while)\b/,function:/[a-z_]\w*(?=\s*\()/i,number:/(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i,operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/}),a.languages.insertBefore("c","string",{macro:{pattern:/(^\s*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im,lookbehind:!0,greedy:!0,alias:"property",inside:{string:[{pattern:/^(#\s*include\s*)<[^>]+>/,lookbehind:!0},a.languages.c.string],comment:a.languages.c.comment,"macro-name":[{pattern:/(^#\s*define\s+)\w+\b(?!\()/i,lookbehind:!0},{pattern:/(^#\s*define\s+)\w+\b(?=\()/i,lookbehind:!0,alias:"function"}],directive:{pattern:/^(#\s*)[a-z]+/,lookbehind:!0,alias:"keyword"},"directive-hash":/^#/,punctuation:/##|\\(?=[\r\n])/,expression:{pattern:/\S[\s\S]*/,inside:a.languages.c}}},constant:/\b(?:__FILE__|__LINE__|__DATE__|__TIME__|__TIMESTAMP__|__func__|EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|stdin|stdout|stderr)\b/}),delete a.languages.c.boolean,function(e){var n=/\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char8_t|char16_t|char32_t|class|compl|concept|const|consteval|constexpr|constinit|const_cast|continue|co_await|co_return|co_yield|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|float|for|friend|goto|if|inline|int|int8_t|int16_t|int32_t|int64_t|uint8_t|uint16_t|uint32_t|uint64_t|long|mutable|namespace|new|noexcept|nullptr|operator|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/;e.languages.cpp=e.languages.extend("c",{"class-name":[{pattern:RegExp(/(\b(?:class|concept|enum|struct|typename)\s+)(?!<keyword>)\w+/.source.replace(/<keyword>/g,(function(){return n.source}))),lookbehind:!0},/\b[A-Z]\w*(?=\s*::\s*\w+\s*\()/,/\b[A-Z_]\w*(?=\s*::\s*~\w+\s*\()/i,/\w+(?=\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*::\s*\w+\s*\()/],keyword:n,number:{pattern:/(?:\b0b[01']+|\b0x(?:[\da-f']+(?:\.[\da-f']*)?|\.[\da-f']+)(?:p[+-]?[\d']+)?|(?:\b[\d']+(?:\.[\d']*)?|\B\.[\d']+)(?:e[+-]?[\d']+)?)[ful]{0,4}/i,greedy:!0},operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/,boolean:/\b(?:true|false)\b/}),e.languages.insertBefore("cpp","string",{"raw-string":{pattern:/R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/,alias:"string",greedy:!0}}),e.languages.insertBefore("cpp","class-name",{"base-clause":{pattern:/(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/,lookbehind:!0,greedy:!0,inside:e.languages.extend("cpp",{})}}),e.languages.insertBefore("inside","operator",{"class-name":/\b[a-z_]\w*\b(?!\s*::)/i},e.languages.cpp["base-clause"])}(a),function(e){var n=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-](?:[^;{\s]|\s+(?![\s{]))*(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+n.source+"|"+/(?:[^\\\r\n()"']|\\[\s\S])*/.source+")\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+n.source+"$"),alias:"url"}}},selector:RegExp("[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+n.source+")*(?=\\s*\\{)"),string:{pattern:n,greedy:!0},property:/(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,important:/!important\b/i,function:/[-a-z0-9]+(?=\()/i,punctuation:/[(){};:,]/},e.languages.css.atrule.inside.rest=e.languages.css;var t=e.languages.markup;t&&(t.tag.addInlined("style","css"),e.languages.insertBefore("inside","attr-value",{"style-attr":{pattern:/(^|["'\s])style\s*=\s*(?:"[^"]*"|'[^']*')/i,lookbehind:!0,inside:{"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{style:{pattern:/(["'])[\s\S]+(?=["']$)/,lookbehind:!0,alias:"language-css",inside:e.languages.css},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},"attr-name":/^style/i}}},t.tag))}(a),function(e){var n,t=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css.selector={pattern:e.languages.css.selector,inside:n={"pseudo-element":/:(?:after|before|first-letter|first-line|selection)|::[-\w]+/,"pseudo-class":/:[-\w]+/,class:/\.[-\w]+/,id:/#[-\w]+/,attribute:{pattern:RegExp("\\[(?:[^[\\]\"']|"+t.source+")*\\]"),greedy:!0,inside:{punctuation:/^\[|\]$/,"case-sensitivity":{pattern:/(\s)[si]$/i,lookbehind:!0,alias:"keyword"},namespace:{pattern:/^(\s*)(?:(?!\s)[-*\w\xA0-\uFFFF])*\|(?!=)/,lookbehind:!0,inside:{punctuation:/\|$/}},"attr-name":{pattern:/^(\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+/,lookbehind:!0},"attr-value":[t,{pattern:/(=\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+(?=\s*$)/,lookbehind:!0}],operator:/[|~*^$]?=/}},"n-th":[{pattern:/(\(\s*)[+-]?\d*[\dn](?:\s*[+-]\s*\d+)?(?=\s*\))/,lookbehind:!0,inside:{number:/[\dn]+/,operator:/[+-]/}},{pattern:/(\(\s*)(?:even|odd)(?=\s*\))/i,lookbehind:!0}],combinator:/>|\+|~|\|\|/,punctuation:/[(),]/}},e.languages.css.atrule.inside["selector-function-argument"].inside=n,e.languages.insertBefore("css","property",{variable:{pattern:/(^|[^-\w\xA0-\uFFFF])--(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*/i,lookbehind:!0}});var i={pattern:/(\b\d+)(?:%|[a-z]+\b)/,lookbehind:!0},r={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0};e.languages.insertBefore("css","function",{operator:{pattern:/(\s)[+\-*\/](?=\s)/,lookbehind:!0},hexcode:{pattern:/\B#(?:[\da-f]{1,2}){3,4}\b/i,alias:"color"},color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:rgb|hsl)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:rgb|hsl)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:i,number:r,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:i,number:r})}(a),a.languages.javascript=a.languages.extend("clike",{"class-name":[a.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:prototype|constructor))/,lookbehind:!0}],keyword:[{pattern:/((?:^|})\s*)(?:catch|finally)\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|for|from|function|(?:get|set)(?=\s*[\[$\w\xA0-\uFFFF])|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:/\b(?:(?:0[xX](?:[\dA-Fa-f](?:_[\dA-Fa-f])?)+|0[bB](?:[01](?:_[01])?)+|0[oO](?:[0-7](?:_[0-7])?)+)n?|(?:\d(?:_\d)?)+n|NaN|Infinity)\b|(?:\b(?:\d(?:_\d)?)+\.?(?:\d(?:_\d)?)*|\B\.(?:\d(?:_\d)?)+)(?:[Ee][+-]?(?:\d(?:_\d)?)+)?/,operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),a.languages.javascript["class-name"][0].pattern=/(\b(?:class|interface|extends|implements|instanceof|new)\s+)[\w.\\]+/,a.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*]|\\.|[^/\\\[\r\n])+\/[gimyus]{0,6}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:a.languages.regex},"regex-flags":/[a-z]+$/,"regex-delimiter":/^\/|\/$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:a.languages.javascript},{pattern:/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,inside:a.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:a.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:a.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),a.languages.insertBefore("javascript","string",{"template-string":{pattern:/`(?:\\[\s\S]|\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}|(?!\${)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\${|}$/,alias:"punctuation"},rest:a.languages.javascript}},string:/[\s\S]+/}}}),a.languages.markup&&a.languages.markup.tag.addInlined("script","javascript"),a.languages.js=a.languages.javascript,function(e){var n=e.util.clone(e.languages.javascript);e.languages.jsx=e.languages.extend("markup",n),e.languages.jsx.tag.pattern=/<\/?(?:[\w.:-]+(?:\s+(?:[\w.:$-]+(?:=(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s{'">=]+|\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}))?|\{\s*\.{3}\s*[a-z_$][\w$]*(?:\.[a-z_$][\w$]*)*\s*\}))*\s*\/?)?>/i,e.languages.jsx.tag.inside.tag.pattern=/^<\/?[^\s>\/]*/i,e.languages.jsx.tag.inside["attr-value"].pattern=/=(?!\{)(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s'">]+)/i,e.languages.jsx.tag.inside.tag.inside["class-name"]=/^[A-Z]\w*(?:\.[A-Z]\w*)*$/,e.languages.insertBefore("inside","attr-name",{spread:{pattern:/\{\s*\.{3}\s*[a-z_$][\w$]*(?:\.[a-z_$][\w$]*)*\s*\}/,inside:{punctuation:/\.{3}|[{}.]/,"attr-value":/\w+/}}},e.languages.jsx.tag),e.languages.insertBefore("inside","attr-value",{script:{pattern:/=(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\})/i,inside:{"script-punctuation":{pattern:/^=(?={)/,alias:"punctuation"},rest:e.languages.jsx},alias:"language-javascript"}},e.languages.jsx.tag);var t=function(e){return e?"string"==typeof e?e:"string"==typeof e.content?e.content:e.content.map(t).join(""):""},i=function(n){for(var r=[],o=0;o<n.length;o++){var a=n[o],l=!1;if("string"!=typeof a&&("tag"===a.type&&a.content[0]&&"tag"===a.content[0].type?"</"===a.content[0].content[0].content?r.length>0&&r[r.length-1].tagName===t(a.content[0].content[1])&&r.pop():"/>"===a.content[a.content.length-1].content||r.push({tagName:t(a.content[0].content[1]),openedBraces:0}):r.length>0&&"punctuation"===a.type&&"{"===a.content?r[r.length-1].openedBraces++:r.length>0&&r[r.length-1].openedBraces>0&&"punctuation"===a.type&&"}"===a.content?r[r.length-1].openedBraces--:l=!0),(l||"string"==typeof a)&&r.length>0&&0===r[r.length-1].openedBraces){var s=t(a);o<n.length-1&&("string"==typeof n[o+1]||"plain-text"===n[o+1].type)&&(s+=t(n[o+1]),n.splice(o+1,1)),o>0&&("string"==typeof n[o-1]||"plain-text"===n[o-1].type)&&(s=t(n[o-1])+s,n.splice(o-1,1),o--),n[o]=new e.Token("plain-text",s,null,s)}a.content&&"string"!=typeof a.content&&i(a.content)}};e.hooks.add("after-tokenize",(function(e){"jsx"!==e.language&&"tsx"!==e.language||i(e.tokens)}))}(a),function(e){function n(e,n){return RegExp(e.replace(/<ID>/g,(function(){return/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/.source})),n)}e.languages.insertBefore("javascript","function-variable",{"method-variable":{pattern:RegExp("(\\.\\s*)"+e.languages.javascript["function-variable"].pattern.source),lookbehind:!0,alias:["function-variable","method","function","property-access"]}}),e.languages.insertBefore("javascript","function",{method:{pattern:RegExp("(\\.\\s*)"+e.languages.javascript.function.source),lookbehind:!0,alias:["function","property-access"]}}),e.languages.insertBefore("javascript","constant",{"known-class-name":[{pattern:/\b(?:(?:(?:Uint|Int)(?:8|16|32)|Uint8Clamped|Float(?:32|64))?Array|ArrayBuffer|BigInt|Boolean|DataView|Date|Error|Function|Intl|JSON|Math|Number|Object|Promise|Proxy|Reflect|RegExp|String|Symbol|(?:Weak)?(?:Set|Map)|WebAssembly)\b/,alias:"class-name"},{pattern:/\b(?:[A-Z]\w*)Error\b/,alias:"class-name"}]}),e.languages.insertBefore("javascript","keyword",{imports:{pattern:n(/(\bimport\b\s*)(?:<ID>(?:\s*,\s*(?:\*\s*as\s+<ID>|\{[^{}]*\}))?|\*\s*as\s+<ID>|\{[^{}]*\})(?=\s*\bfrom\b)/.source),lookbehind:!0,inside:e.languages.javascript},exports:{pattern:n(/(\bexport\b\s*)(?:\*(?:\s*as\s+<ID>)?(?=\s*\bfrom\b)|\{[^{}]*\})/.source),lookbehind:!0,inside:e.languages.javascript}}),e.languages.javascript.keyword.unshift({pattern:/\b(?:as|default|export|from|import)\b/,alias:"module"},{pattern:/\b(?:await|break|catch|continue|do|else|for|finally|if|return|switch|throw|try|while|yield)\b/,alias:"control-flow"},{pattern:/\bnull\b/,alias:["null","nil"]},{pattern:/\bundefined\b/,alias:"nil"}),e.languages.insertBefore("javascript","operator",{spread:{pattern:/\.{3}/,alias:"operator"},arrow:{pattern:/=>/,alias:"operator"}}),e.languages.insertBefore("javascript","punctuation",{"property-access":{pattern:n(/(\.\s*)#?<ID>/.source),lookbehind:!0},"maybe-class-name":{pattern:/(^|[^$\w\xA0-\uFFFF])[A-Z][$\w\xA0-\uFFFF]+/,lookbehind:!0},dom:{pattern:/\b(?:document|location|navigator|performance|(?:local|session)Storage|window)\b/,alias:"variable"},console:{pattern:/\bconsole(?=\s*\.)/,alias:"class-name"}});for(var t=["function","function-variable","method","method-variable","property-access"],i=0;i<t.length;i++){var r=t[i],o=e.languages.javascript[r];"RegExp"===e.util.type(o)&&(o=e.languages.javascript[r]={pattern:o});var a=o.inside||{};o.inside=a,a["maybe-class-name"]=/^[A-Z][\s\S]*/}}(a),function(e){var n=/#(?!\{).+/,t={pattern:/#\{[^}]+\}/,alias:"variable"};e.languages.coffeescript=e.languages.extend("javascript",{comment:n,string:[{pattern:/'(?:\\[\s\S]|[^\\'])*'/,greedy:!0},{pattern:/"(?:\\[\s\S]|[^\\"])*"/,greedy:!0,inside:{interpolation:t}}],keyword:/\b(?:and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield)\b/,"class-member":{pattern:/@(?!\d)\w+/,alias:"variable"}}),e.languages.insertBefore("coffeescript","comment",{"multiline-comment":{pattern:/###[\s\S]+?###/,alias:"comment"},"block-regex":{pattern:/\/{3}[\s\S]*?\/{3}/,alias:"regex",inside:{comment:n,interpolation:t}}}),e.languages.insertBefore("coffeescript","string",{"inline-javascript":{pattern:/`(?:\\[\s\S]|[^\\`])*`/,inside:{delimiter:{pattern:/^`|`$/,alias:"punctuation"},script:{pattern:/[\s\S]+/,alias:"language-javascript",inside:e.languages.javascript}}},"multiline-string":[{pattern:/'''[\s\S]*?'''/,greedy:!0,alias:"string"},{pattern:/"""[\s\S]*?"""/,greedy:!0,alias:"string",inside:{interpolation:t}}]}),e.languages.insertBefore("coffeescript","keyword",{property:/(?!\d)\w+(?=\s*:(?!:))/}),delete e.languages.coffeescript["template-string"],e.languages.coffee=e.languages.coffeescript}(a),function(e){e.languages.diff={coord:[/^(?:\*{3}|-{3}|\+{3}).*$/m,/^@@.*@@$/m,/^\d.*$/m]};var n={"deleted-sign":"-","deleted-arrow":"<","inserted-sign":"+","inserted-arrow":">",unchanged:" ",diff:"!"};Object.keys(n).forEach((function(t){var i=n[t],r=[];/^\w+$/.test(t)||r.push(/\w+/.exec(t)[0]),"diff"===t&&r.push("bold"),e.languages.diff[t]={pattern:RegExp("^(?:["+i+"].*(?:\r\n?|\n|(?![\\s\\S])))+","m"),alias:r,inside:{line:{pattern:/(.)(?=[\s\S]).*(?:\r\n?|\n)?/,lookbehind:!0},prefix:{pattern:/[\s\S]/,alias:/\w+/.exec(t)[0]}}}})),Object.defineProperty(e.languages.diff,"PREFIXES",{value:n})}(a),a.languages.git={comment:/^#.*/m,deleted:/^[-\u2013].*/m,inserted:/^\+.*/m,string:/("|')(?:\\.|(?!\1)[^\\\r\n])*\1/m,command:{pattern:/^.*\$ git .*$/m,inside:{parameter:/\s--?\w+/m}},coord:/^@@.*@@$/m,"commit-sha1":/^commit \w{40}$/m},a.languages.go=a.languages.extend("clike",{string:{pattern:/(["'`])(?:\\[\s\S]|(?!\1)[^\\])*\1/,greedy:!0},keyword:/\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,boolean:/\b(?:_|iota|nil|true|false)\b/,number:/(?:\b0x[a-f\d]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[-+]?\d+)?)i?/i,operator:/[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,builtin:/\b(?:bool|byte|complex(?:64|128)|error|float(?:32|64)|rune|string|u?int(?:8|16|32|64)?|uintptr|append|cap|close|complex|copy|delete|imag|len|make|new|panic|print(?:ln)?|real|recover)\b/}),delete a.languages.go["class-name"],a.languages.graphql={comment:/#.*/,description:{pattern:/(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i,greedy:!0,alias:"string",inside:{"language-markdown":{pattern:/(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/,lookbehind:!0,inside:a.languages.markdown}}},string:{pattern:/"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/,greedy:!0},number:/(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,boolean:/\b(?:true|false)\b/,variable:/\$[a-z_]\w*/i,directive:{pattern:/@[a-z_]\w*/i,alias:"function"},"attr-name":{pattern:/[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i,greedy:!0},"class-name":{pattern:/(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*)[a-zA-Z_]\w*/,lookbehind:!0},fragment:{pattern:/(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},keyword:/\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/,operator:/[!=|&]|\.{3}/,punctuation:/[!(){}\[\]:=,]/,constant:/\b(?!ID\b)[A-Z][A-Z_\d]*\b/},function(e){function n(e,n){return"___"+e.toUpperCase()+n+"___"}Object.defineProperties(e.languages["markup-templating"]={},{buildPlaceholders:{value:function(t,i,r,o){if(t.language===i){var a=t.tokenStack=[];t.code=t.code.replace(r,(function(e){if("function"==typeof o&&!o(e))return e;for(var r,l=a.length;-1!==t.code.indexOf(r=n(i,l));)++l;return a[l]=e,r})),t.grammar=e.languages.markup}}},tokenizePlaceholders:{value:function(t,i){if(t.language===i&&t.tokenStack){t.grammar=e.languages[i];var r=0,o=Object.keys(t.tokenStack);!function a(l){for(var s=0;s<l.length&&!(r>=o.length);s++){var c=l[s];if("string"==typeof c||c.content&&"string"==typeof c.content){var u=o[r],p=t.tokenStack[u],d="string"==typeof c?c:c.content,f=n(i,u),m=d.indexOf(f);if(m>-1){++r;var g=d.substring(0,m),h=new e.Token(i,e.tokenize(p,t.grammar),"language-"+i,p),v=d.substring(m+f.length),b=[];g&&b.push.apply(b,a([g])),b.push(h),v&&b.push.apply(b,a([v])),"string"==typeof c?l.splice.apply(l,[s,1].concat(b)):c.content=b}}else c.content&&a(c.content)}return l}(t.tokens)}}}})}(a),function(e){e.languages.handlebars={comment:/\{\{![\s\S]*?\}\}/,delimiter:{pattern:/^\{\{\{?|\}\}\}?$/i,alias:"punctuation"},string:/(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][+-]?\d+)?/,boolean:/\b(?:true|false)\b/,block:{pattern:/^(\s*(?:~\s*)?)[#\/]\S+?(?=\s*(?:~\s*)?$|\s)/i,lookbehind:!0,alias:"keyword"},brackets:{pattern:/\[[^\]]+\]/,inside:{punctuation:/\[|\]/,variable:/[\s\S]+/}},punctuation:/[!"#%&':()*+,.\/;<=>@\[\\\]^`{|}~]/,variable:/[^!"#%&'()*+,\/;<=>@\[\\\]^`{|}~\s]+/},e.hooks.add("before-tokenize",(function(n){e.languages["markup-templating"].buildPlaceholders(n,"handlebars",/\{\{\{[\s\S]+?\}\}\}|\{\{[\s\S]+?\}\}/g)})),e.hooks.add("after-tokenize",(function(n){e.languages["markup-templating"].tokenizePlaceholders(n,"handlebars")}))}(a),a.languages.json={property:{pattern:/"(?:\\.|[^\\"\r\n])*"(?=\s*:)/,greedy:!0},string:{pattern:/"(?:\\.|[^\\"\r\n])*"(?!\s*:)/,greedy:!0},comment:{pattern:/\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},number:/-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,punctuation:/[{}[\],]/,operator:/:/,boolean:/\b(?:true|false)\b/,null:{pattern:/\bnull\b/,alias:"keyword"}},a.languages.webmanifest=a.languages.json,a.languages.less=a.languages.extend("css",{comment:[/\/\*[\s\S]*?\*\//,{pattern:/(^|[^\\])\/\/.*/,lookbehind:!0}],atrule:{pattern:/@[\w-](?:\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{punctuation:/[:()]/}},selector:{pattern:/(?:@\{[\w-]+\}|[^{};\s@])(?:@\{[\w-]+\}|\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};@\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{variable:/@+[\w-]+/}},property:/(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/i,operator:/[+\-*\/]/}),a.languages.insertBefore("less","property",{variable:[{pattern:/@[\w-]+\s*:/,inside:{punctuation:/:/}},/@@?[\w-]+/],"mixin-usage":{pattern:/([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/,lookbehind:!0,alias:"function"}}),a.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|[^\\\r\n])*/,lookbehind:!0},string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},builtin:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,symbol:{pattern:/^(?:[^:=\s]|[ \t]+(?![\s:]))+(?=\s*:(?!=))/m,inside:{variable:/\$+(?:(?!\$)[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:(?!\$)[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:[/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,{pattern:/(\()(?:addsuffix|abspath|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:s|list)?)(?=[ \t])/,lookbehind:!0}],operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/},function(e){var n=/(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?!\n|\r\n?))/.source;function t(e){return e=e.replace(/<inner>/g,(function(){return n})),RegExp(/((?:^|[^\\])(?:\\{2})*)/.source+"(?:"+e+")")}var i=/(?:\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\|\r\n`])+/.source,r=/\|?__(?:\|__)+\|?(?:(?:\n|\r\n?)|(?![\s\S]))/.source.replace(/__/g,(function(){return i})),o=/\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/.source;e.languages.markdown=e.languages.extend("markup",{}),e.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"font-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:e.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+r+o+"(?:"+r+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+r+o+")(?:"+r+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(i),inside:e.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+r+")"+o+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+r+"$"),inside:{"table-header":{pattern:RegExp(i),alias:"important",inside:e.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/``.+?``|`[^`\r\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:t(/\b__(?:(?!_)<inner>|_(?:(?!_)<inner>)+_)+__\b|\*\*(?:(?!\*)<inner>|\*(?:(?!\*)<inner>)+\*)+\*\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:t(/\b_(?:(?!_)<inner>|__(?:(?!_)<inner>)+__)+_\b|\*(?:(?!\*)<inner>|\*\*(?:(?!\*)<inner>)+\*\*)+\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:t(/(~~?)(?:(?!~)<inner>)+?\2/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:t(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(n){["url","bold","italic","strike"].forEach((function(t){n!==t&&(e.languages.markdown[n].inside.content.inside[t]=e.languages.markdown[t])}))})),e.hooks.add("after-tokenize",(function(e){"markdown"!==e.language&&"md"!==e.language||function e(n){if(n&&"string"!=typeof n)for(var t=0,i=n.length;t<i;t++){var r=n[t];if("code"===r.type){var o=r.content[1],a=r.content[3];if(o&&a&&"code-language"===o.type&&"code-block"===a.type&&"string"==typeof o.content){var l=o.content.replace(/\b#/g,"sharp").replace(/\b\+\+/g,"pp"),s="language-"+(l=(/[a-z][\w-]*/i.exec(l)||[""])[0].toLowerCase());a.alias?"string"==typeof a.alias?a.alias=[a.alias,s]:a.alias.push(s):a.alias=[s]}}else e(r.content)}}(e.tokens)})),e.hooks.add("wrap",(function(n){if("code-block"===n.type){for(var t="",i=0,r=n.classes.length;i<r;i++){var o=n.classes[i],a=/language-(.+)/.exec(o);if(a){t=a[1];break}}var l=e.languages[t];if(l){var s=n.content.replace(/&lt;/g,"<").replace(/&amp;/g,"&");n.content=e.highlight(s,l,t)}else if(t&&"none"!==t&&e.plugins.autoloader){var c="md-"+(new Date).valueOf()+"-"+Math.floor(1e16*Math.random());n.attributes.id=c,e.plugins.autoloader.loadLanguages(t,(function(){var n=document.getElementById(c);n&&(n.innerHTML=e.highlight(n.textContent,e.languages[t],t))}))}}})),e.languages.md=e.languages.markdown}(a),a.languages.objectivec=a.languages.extend("c",{string:/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1|@"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,keyword:/\b(?:asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|in|self|super)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/,operator:/-[->]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|\|?|[~^%?*\/@]/}),delete a.languages.objectivec["class-name"],a.languages.objc=a.languages.objectivec,a.languages.ocaml={comment:/\(\*[\s\S]*?\*\)/,string:[{pattern:/"(?:\\.|[^\\\r\n"])*"/,greedy:!0},{pattern:/(['`])(?:\\(?:\d+|x[\da-f]+|.)|(?!\1)[^\\\r\n])\1/i,greedy:!0}],number:/\b(?:0x[\da-f][\da-f_]+|(?:0[bo])?\d[\d_]*(?:\.[\d_]*)?(?:e[+-]?[\d_]+)?)/i,directive:{pattern:/\B#\w+/,alias:"important"},label:{pattern:/\B~\w+/,alias:"function"},"type-variable":{pattern:/\B'\w+/,alias:"function"},variant:{pattern:/`\w+/,alias:"variable"},module:{pattern:/\b[A-Z]\w+/,alias:"variable"},keyword:/\b(?:as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|module|mutable|new|nonrec|object|of|open|private|rec|sig|struct|then|to|try|type|val|value|virtual|when|where|while|with)\b/,boolean:/\b(?:false|true)\b/,operator:/:=|[=<>@^|&+\-*\/$%!?~][!$%&*+\-.\/:<=>?@^|~]*|\b(?:and|asr|land|lor|lsl|lsr|lxor|mod|or)\b/,punctuation:/[(){}\[\]|.,:;]|\b_\b/},a.languages.python={comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0},"string-interpolation":{pattern:/(?:f|rf|fr)(?:("""|''')[\s\S]*?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,greedy:!0,inside:{interpolation:{pattern:/((?:^|[^{])(?:{{)*){(?!{)(?:[^{}]|{(?!{)(?:[^{}]|{(?!{)(?:[^{}])+})+})+}/,lookbehind:!0,inside:{"format-spec":{pattern:/(:)[^:(){}]+(?=}$)/,lookbehind:!0},"conversion-option":{pattern:/![sra](?=[:}]$)/,alias:"punctuation"},rest:null}},string:/[\s\S]+/}},"triple-quoted-string":{pattern:/(?:[rub]|rb|br)?("""|''')[\s\S]*?\1/i,greedy:!0,alias:"string"},string:{pattern:/(?:[rub]|rb|br)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,greedy:!0},function:{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)\w+/i,lookbehind:!0},decorator:{pattern:/(^\s*)@\w+(?:\.\w+)*/im,lookbehind:!0,alias:["annotation","punctuation"],inside:{punctuation:/\./}},keyword:/\b(?:and|as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|print|raise|return|try|while|with|yield)\b/,builtin:/\b(?:__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip)\b/,boolean:/\b(?:True|False|None)\b/,number:/(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?j?\b/i,operator:/[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,punctuation:/[{}[\];(),.:]/},a.languages.python["string-interpolation"].inside.interpolation.inside.rest=a.languages.python,a.languages.py=a.languages.python,a.languages.reason=a.languages.extend("clike",{string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/,greedy:!0},"class-name":/\b[A-Z]\w*/,keyword:/\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/,operator:/\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:mod|land|lor|lxor|lsl|lsr|asr)\b/}),a.languages.insertBefore("reason","class-name",{character:{pattern:/'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/,alias:"string"},constructor:{pattern:/\b[A-Z]\w*\b(?!\s*\.)/,alias:"variable"},label:{pattern:/\b[a-z]\w*(?=::)/,alias:"symbol"}}),delete a.languages.reason.function,function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m,lookbehind:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,inside:{atrule:/(?:@[\w-]+|[+=])/m}}}),delete e.languages.sass.atrule;var n=/\$[-\w]+|#\{\$[-\w]+\}/,t=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|or|not)\b/,{pattern:/(\s+)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,inside:{punctuation:/:/,variable:n,operator:t}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:n,operator:t,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/,lookbehind:!0}})}(a),a.languages.scss=a.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-](?:\([^()]+\)|[^()\s]|\s+(?!\s))*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)?url(?=\()/i,selector:{pattern:/(?=\S)[^@;{}()]?(?:[^@;{}()\s]|\s+(?!\s)|#\{\$[-\w]+\})+(?=\s*\{(?:\}|\s|[^}][^:{}]*[:{][^}]+))/m,inside:{parent:{pattern:/&/,alias:"important"},placeholder:/%[-\w]+/,variable:/\$[-\w]+|#\{\$[-\w]+\}/}},property:{pattern:/(?:[-\w]|\$[-\w]|#\{\$[-\w]+\})+(?=\s*:)/,inside:{variable:/\$[-\w]+|#\{\$[-\w]+\}/}}}),a.languages.insertBefore("scss","atrule",{keyword:[/@(?:if|else(?: if)?|forward|for|each|while|import|use|extend|debug|warn|mixin|include|function|return|content)\b/i,{pattern:/( +)(?:from|through)(?= )/,lookbehind:!0}]}),a.languages.insertBefore("scss","important",{variable:/\$[-\w]+|#\{\$[-\w]+\}/}),a.languages.insertBefore("scss","function",{"module-modifier":{pattern:/\b(?:as|with|show|hide)\b/i,alias:"keyword"},placeholder:{pattern:/%[-\w]+/,alias:"selector"},statement:{pattern:/\B!(?:default|optional)\b/i,alias:"keyword"},boolean:/\b(?:true|false)\b/,null:{pattern:/\bnull\b/,alias:"keyword"},operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|or|not)(?=\s)/,lookbehind:!0}}),a.languages.scss.atrule.inside.rest=a.languages.scss,a.languages.sql={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,lookbehind:!0},variable:[{pattern:/@(["'`])(?:\\[\s\S]|(?!\1)[^\\])+\1/,greedy:!0},/@[\w.$]+/],string:{pattern:/(^|[^@\\])("|')(?:\\[\s\S]|(?!\2)[^\\]|\2\2)*\2/,greedy:!0,lookbehind:!0},function:/\b(?:AVG|COUNT|FIRST|FORMAT|LAST|LCASE|LEN|MAX|MID|MIN|MOD|NOW|ROUND|SUM|UCASE)(?=\s*\()/i,keyword:/\b(?:ACTION|ADD|AFTER|ALGORITHM|ALL|ALTER|ANALYZE|ANY|APPLY|AS|ASC|AUTHORIZATION|AUTO_INCREMENT|BACKUP|BDB|BEGIN|BERKELEYDB|BIGINT|BINARY|BIT|BLOB|BOOL|BOOLEAN|BREAK|BROWSE|BTREE|BULK|BY|CALL|CASCADED?|CASE|CHAIN|CHAR(?:ACTER|SET)?|CHECK(?:POINT)?|CLOSE|CLUSTERED|COALESCE|COLLATE|COLUMNS?|COMMENT|COMMIT(?:TED)?|COMPUTE|CONNECT|CONSISTENT|CONSTRAINT|CONTAINS(?:TABLE)?|CONTINUE|CONVERT|CREATE|CROSS|CURRENT(?:_DATE|_TIME|_TIMESTAMP|_USER)?|CURSOR|CYCLE|DATA(?:BASES?)?|DATE(?:TIME)?|DAY|DBCC|DEALLOCATE|DEC|DECIMAL|DECLARE|DEFAULT|DEFINER|DELAYED|DELETE|DELIMITERS?|DENY|DESC|DESCRIBE|DETERMINISTIC|DISABLE|DISCARD|DISK|DISTINCT|DISTINCTROW|DISTRIBUTED|DO|DOUBLE|DROP|DUMMY|DUMP(?:FILE)?|DUPLICATE|ELSE(?:IF)?|ENABLE|ENCLOSED|END|ENGINE|ENUM|ERRLVL|ERRORS|ESCAPED?|EXCEPT|EXEC(?:UTE)?|EXISTS|EXIT|EXPLAIN|EXTENDED|FETCH|FIELDS|FILE|FILLFACTOR|FIRST|FIXED|FLOAT|FOLLOWING|FOR(?: EACH ROW)?|FORCE|FOREIGN|FREETEXT(?:TABLE)?|FROM|FULL|FUNCTION|GEOMETRY(?:COLLECTION)?|GLOBAL|GOTO|GRANT|GROUP|HANDLER|HASH|HAVING|HOLDLOCK|HOUR|IDENTITY(?:_INSERT|COL)?|IF|IGNORE|IMPORT|INDEX|INFILE|INNER|INNODB|INOUT|INSERT|INT|INTEGER|INTERSECT|INTERVAL|INTO|INVOKER|ISOLATION|ITERATE|JOIN|KEYS?|KILL|LANGUAGE|LAST|LEAVE|LEFT|LEVEL|LIMIT|LINENO|LINES|LINESTRING|LOAD|LOCAL|LOCK|LONG(?:BLOB|TEXT)|LOOP|MATCH(?:ED)?|MEDIUM(?:BLOB|INT|TEXT)|MERGE|MIDDLEINT|MINUTE|MODE|MODIFIES|MODIFY|MONTH|MULTI(?:LINESTRING|POINT|POLYGON)|NATIONAL|NATURAL|NCHAR|NEXT|NO|NONCLUSTERED|NULLIF|NUMERIC|OFF?|OFFSETS?|ON|OPEN(?:DATASOURCE|QUERY|ROWSET)?|OPTIMIZE|OPTION(?:ALLY)?|ORDER|OUT(?:ER|FILE)?|OVER|PARTIAL|PARTITION|PERCENT|PIVOT|PLAN|POINT|POLYGON|PRECEDING|PRECISION|PREPARE|PREV|PRIMARY|PRINT|PRIVILEGES|PROC(?:EDURE)?|PUBLIC|PURGE|QUICK|RAISERROR|READS?|REAL|RECONFIGURE|REFERENCES|RELEASE|RENAME|REPEAT(?:ABLE)?|REPLACE|REPLICATION|REQUIRE|RESIGNAL|RESTORE|RESTRICT|RETURN(?:S|ING)?|REVOKE|RIGHT|ROLLBACK|ROUTINE|ROW(?:COUNT|GUIDCOL|S)?|RTREE|RULE|SAVE(?:POINT)?|SCHEMA|SECOND|SELECT|SERIAL(?:IZABLE)?|SESSION(?:_USER)?|SET(?:USER)?|SHARE|SHOW|SHUTDOWN|SIMPLE|SMALLINT|SNAPSHOT|SOME|SONAME|SQL|START(?:ING)?|STATISTICS|STATUS|STRIPED|SYSTEM_USER|TABLES?|TABLESPACE|TEMP(?:ORARY|TABLE)?|TERMINATED|TEXT(?:SIZE)?|THEN|TIME(?:STAMP)?|TINY(?:BLOB|INT|TEXT)|TOP?|TRAN(?:SACTIONS?)?|TRIGGER|TRUNCATE|TSEQUAL|TYPES?|UNBOUNDED|UNCOMMITTED|UNDEFINED|UNION|UNIQUE|UNLOCK|UNPIVOT|UNSIGNED|UPDATE(?:TEXT)?|USAGE|USE|USER|USING|VALUES?|VAR(?:BINARY|CHAR|CHARACTER|YING)|VIEW|WAITFOR|WARNINGS|WHEN|WHERE|WHILE|WITH(?: ROLLUP|IN)?|WORK|WRITE(?:TEXT)?|YEAR)\b/i,boolean:/\b(?:TRUE|FALSE|NULL)\b/i,number:/\b0x[\da-f]+\b|\b\d+(?:\.\d*)?|\B\.\d+\b/i,operator:/[-+*\/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?|\b(?:AND|BETWEEN|IN|LIKE|NOT|OR|IS|DIV|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b/i,punctuation:/[;[\]()`,.]/},function(e){var n={pattern:/(\b\d+)(?:%|[a-z]+)/,lookbehind:!0},t={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0},i={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},url:{pattern:/url\((["']?).*?\1\)/i,greedy:!0},string:{pattern:/("|')(?:(?!\1)[^\\\r\n]|\\(?:\r\n|[\s\S]))*\1/,greedy:!0},interpolation:null,func:null,important:/\B!(?:important|optional)\b/i,keyword:{pattern:/(^|\s+)(?:(?:if|else|for|return|unless)(?=\s+|$)|@[\w-]+)/,lookbehind:!0},hexcode:/#[\da-f]{3,6}/i,color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:rgb|hsl)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:rgb|hsl)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:n,number:t,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:n,boolean:/\b(?:true|false)\b/,operator:[/~|[+!\/%<>?=]=?|[-:]=|\*[*=]?|\.{2,3}|&&|\|\||\B-\B|\b(?:and|in|is(?: a| defined| not|nt)?|not|or)\b/],number:t,punctuation:/[{}()\[\];:,]/};i.interpolation={pattern:/\{[^\r\n}:]+\}/,alias:"variable",inside:{delimiter:{pattern:/^{|}$/,alias:"punctuation"},rest:i}},i.func={pattern:/[\w-]+\([^)]*\).*/,inside:{function:/^[^(]+/,rest:i}},e.languages.stylus={"atrule-declaration":{pattern:/(^\s*)@.+/m,lookbehind:!0,inside:{atrule:/^@[\w-]+/,rest:i}},"variable-declaration":{pattern:/(^[ \t]*)[\w$-]+\s*.?=[ \t]*(?:\{[^{}]*\}|\S.*|$)/m,lookbehind:!0,inside:{variable:/^\S+/,rest:i}},statement:{pattern:/(^[ \t]*)(?:if|else|for|return|unless)[ \t].+/m,lookbehind:!0,inside:{keyword:/^\S+/,rest:i}},"property-declaration":{pattern:/((?:^|\{)([ \t]*))(?:[\w-]|\{[^}\r\n]+\})+(?:\s*:\s*|[ \t]+)(?!\s)[^{\r\n]*(?:;|[^{\r\n,](?=$)(?!(?:\r?\n|\r)(?:\{|\2[ \t]+)))/m,lookbehind:!0,inside:{property:{pattern:/^[^\s:]+/,inside:{interpolation:i.interpolation}},rest:i}},selector:{pattern:/(^[ \t]*)(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)(?:(?:\r?\n|\r)(?:\1(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)))*(?:,$|\{|(?=(?:\r?\n|\r)(?:\{|\1[ \t]+)))/m,lookbehind:!0,inside:{interpolation:i.interpolation,comment:i.comment,punctuation:/[{},]/}},func:i.func,string:i.string,comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0,greedy:!0},interpolation:i.interpolation,punctuation:/[{}()\[\];:.]/}}(a),function(e){e.languages.typescript=e.languages.extend("javascript",{"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|type)\s+)(?!keyof\b)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?:\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},keyword:/\b(?:abstract|as|asserts|async|await|break|case|catch|class|const|constructor|continue|debugger|declare|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|is|keyof|let|module|namespace|new|null|of|package|private|protected|public|readonly|return|require|set|static|super|switch|this|throw|try|type|typeof|undefined|var|void|while|with|yield)\b/,builtin:/\b(?:string|Function|any|number|boolean|Array|symbol|console|Promise|unknown|never)\b/}),delete e.languages.typescript.parameter;var n=e.languages.extend("typescript",{});delete n["class-name"],e.languages.typescript["class-name"].inside=n,e.languages.insertBefore("typescript","function",{"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:n}}}}),e.languages.ts=e.languages.typescript}(a),function(e){var n=e.util.clone(e.languages.typescript);e.languages.tsx=e.languages.extend("jsx",n);var t=e.languages.tsx.tag;t.pattern=RegExp(/(^|[^\w$]|(?=<\/))/.source+"(?:"+t.pattern.source+")",t.pattern.flags),t.lookbehind=!0}(a),a.languages.wasm={comment:[/\(;[\s\S]*?;\)/,{pattern:/;;.*/,greedy:!0}],string:{pattern:/"(?:\\[\s\S]|[^"\\])*"/,greedy:!0},keyword:[{pattern:/\b(?:align|offset)=/,inside:{operator:/=/}},{pattern:/\b(?:(?:f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|nearest|neg?|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|store(?:8|16|32)?|sqrt|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))?|memory\.(?:grow|size))\b/,inside:{punctuation:/\./}},/\b(?:anyfunc|block|br(?:_if|_table)?|call(?:_indirect)?|data|drop|elem|else|end|export|func|get_(?:global|local)|global|if|import|local|loop|memory|module|mut|nop|offset|param|result|return|select|set_(?:global|local)|start|table|tee_local|then|type|unreachable)\b/],variable:/\$[\w!#$%&'*+\-./:<=>?@\\^_`|~]+/i,number:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/,punctuation:/[()]/},function(e){var n=/[*&][^\s[\]{},]+/,t=/!(?:<[\w\-%#;/?:@&=+$,.!~*'()[\]]+>|(?:[a-zA-Z\d-]*!)?[\w\-%#;/?:@&=+$.~*'()]+)?/,i="(?:"+t.source+"(?:[ \t]+"+n.source+")?|"+n.source+"(?:[ \t]+"+t.source+")?)",r=/(?:[^\s\x00-\x08\x0e-\x1f!"#%&'*,\-:>?@[\]`{|}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]|[?:-]<PLAIN>)(?:[ \t]*(?:(?![#:])<PLAIN>|:<PLAIN>))*/.source.replace(/<PLAIN>/g,(function(){return/[^\s\x00-\x08\x0e-\x1f,[\]{}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]/.source})),o=/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/.source;function a(e,n){n=(n||"").replace(/m/g,"")+"m";var t=/([:\-,[{]\s*(?:\s<<prop>>[ \t]+)?)(?:<<value>>)(?=[ \t]*(?:$|,|]|}|(?:[\r\n]\s*)?#))/.source.replace(/<<prop>>/g,(function(){return i})).replace(/<<value>>/g,(function(){return e}));return RegExp(t,n)}e.languages.yaml={scalar:{pattern:RegExp(/([\-:]\s*(?:\s<<prop>>[ \t]+)?[|>])[ \t]*(?:((?:\r?\n|\r)[ \t]+)\S[^\r\n]*(?:\2[^\r\n]+)*)/.source.replace(/<<prop>>/g,(function(){return i}))),lookbehind:!0,alias:"string"},comment:/#.*/,key:{pattern:RegExp(/((?:^|[:\-,[{\r\n?])[ \t]*(?:<<prop>>[ \t]+)?)<<key>>(?=\s*:\s)/.source.replace(/<<prop>>/g,(function(){return i})).replace(/<<key>>/g,(function(){return"(?:"+r+"|"+o+")"}))),lookbehind:!0,greedy:!0,alias:"atrule"},directive:{pattern:/(^[ \t]*)%.+/m,lookbehind:!0,alias:"important"},datetime:{pattern:a(/\d{4}-\d\d?-\d\d?(?:[tT]|[ \t]+)\d\d?:\d{2}:\d{2}(?:\.\d*)?(?:[ \t]*(?:Z|[-+]\d\d?(?::\d{2})?))?|\d{4}-\d{2}-\d{2}|\d\d?:\d{2}(?::\d{2}(?:\.\d*)?)?/.source),lookbehind:!0,alias:"number"},boolean:{pattern:a(/true|false/.source,"i"),lookbehind:!0,alias:"important"},null:{pattern:a(/null|~/.source,"i"),lookbehind:!0,alias:"important"},string:{pattern:a(o),lookbehind:!0,greedy:!0},number:{pattern:a(/[+-]?(?:0x[\da-f]+|0o[0-7]+|(?:\d+(?:\.\d*)?|\.?\d+)(?:e[+-]?\d+)?|\.inf|\.nan)/.source,"i"),lookbehind:!0},tag:t,important:n,punctuation:/---|[:[\]{}\-,|>?]|\.\.\./},e.languages.yml=e.languages.yaml}(a),n.a=a},function(e,n,t){"use strict";var i=t(0),r=t.n(i),o=t(35);n.a=function(e){return r.a.createElement(o.a,Object.assign({},e))}},function(e){e.exports=JSON.parse("{}")},function(e,n,t){"use strict";var i=t(9);n.a=i.b},function(e,n){var t;t=function(){return this}();try{t=t||new Function("return this")()}catch(i){"object"==typeof window&&(t=window)}e.exports=t},function(e,n,t){"use strict";e.exports=t(46)},function(e,n,t){"use strict";!function e(){if("undefined"!=typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(n){console.error(n)}}(),e.exports=t(40)},function(e){e.exports=JSON.parse('{"docusaurus-plugin-content-docs":{"default":{"path":"/netsage-pipeline/docs","versions":[{"name":"current","label":"master (unreleased)","isLast":false,"path":"/netsage-pipeline/docs/next","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/next/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/next/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/next/components/docker_pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/next/deploy/docker_install_advanced","sidebar":"Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/next/deploy/docker_install_simple","sidebar":"Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/next/deploy/docker_troubleshoot","sidebar":"Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/next/deploy/docker_upgrade","sidebar":"Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/next/devel/dev_dataset","sidebar":"Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/next/devel/docker_dev_guide","sidebar":"Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/next/devel/docker_dev_tag","sidebar":"Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/next/devel/docusaurus","sidebar":"Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/next/pipeline/elastic","sidebar":"Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/next/pipeline/importer","sidebar":"Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/next/pipeline/intro","sidebar":"Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/next/pipeline/logstash","sidebar":"Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/next/pipeline/nfdump","sidebar":"Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/next/pipeline/tstat","sidebar":"Pipeline"}]},{"name":"2.1.0","label":"2.1.0","isLast":true,"path":"/netsage-pipeline/docs","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/components/docker_pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/deploy/docker_install_advanced","sidebar":"version-2.1.0/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/deploy/docker_install_simple","sidebar":"version-2.1.0/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/deploy/docker_troubleshoot","sidebar":"version-2.1.0/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/deploy/docker_upgrade","sidebar":"version-2.1.0/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/devel/dev_dataset","sidebar":"version-2.1.0/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/devel/docker_dev_guide","sidebar":"version-2.1.0/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/devel/docker_dev_tag","sidebar":"version-2.1.0/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/devel/docusaurus","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/pipeline/elastic","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/pipeline/importer","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/pipeline/intro","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/pipeline/logstash","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/pipeline/nfdump","sidebar":"version-2.1.0/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/pipeline/tstat","sidebar":"version-2.1.0/Pipeline"}]},{"name":"1.2.11","label":"1.2.11","isLast":false,"path":"/netsage-pipeline/docs/1.2.11","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.11/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.11/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.11/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.11/deploy/choose_install","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.11/devel/dev_dataset","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/1.2.11/devel/docusaurus","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.11/pipeline/elastic","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.11/pipeline/importer","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.11/pipeline/intro","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.11/pipeline/logstash","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.11/pipeline/nfdump","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.11/pipeline/tstat","sidebar":"version-1.2.11/Pipeline"}]},{"name":"1.2.10","label":"1.2.10","isLast":false,"path":"/netsage-pipeline/docs/1.2.10","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.10/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.10/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.10/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.10/deploy/choose_install","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.10/devel/dev_dataset","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/1.2.10/devel/docusaurus","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.10/pipeline/elastic","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.10/pipeline/importer","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.10/pipeline/intro","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.10/pipeline/logstash","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.10/pipeline/nfdump","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.10/pipeline/tstat","sidebar":"version-1.2.10/Pipeline"}]},{"name":"1.2.9","label":"1.2.9","isLast":false,"path":"/netsage-pipeline/docs/1.2.9","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.9/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.9/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.9/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.9/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.9/deploy/choose_install","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot","sidebar":"version-1.2.9/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.9/devel/dev_dataset","sidebar":"version-1.2.9/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide","sidebar":"version-1.2.9/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.9/docusaurus","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.9/pipeline/elastic","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.9/pipeline/importer","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.9/pipeline/intro","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.9/pipeline/logstash","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.9/pipeline/nfdump","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.9/pipeline/tstat","sidebar":"version-1.2.9/Pipeline"}]},{"name":"1.2.8","label":"1.2.8","isLast":false,"path":"/netsage-pipeline/docs/1.2.8","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.8/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.8/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.8/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.8/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.8/deploy/choose_install","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot","sidebar":"version-1.2.8/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.8/devel/dev_dataset","sidebar":"version-1.2.8/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide","sidebar":"version-1.2.8/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.8/docusaurus","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.8/pipeline/elastic","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.8/pipeline/importer","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.8/pipeline/intro","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.8/pipeline/logstash","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.8/pipeline/nfdump","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.8/pipeline/tstat","sidebar":"version-1.2.8/Pipeline"}]},{"name":"1.2.7","label":"1.2.7","isLast":false,"path":"/netsage-pipeline/docs/1.2.7","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.7/components/docker_env"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.7/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.7/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.7/deploy/choose_install","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot","sidebar":"version-1.2.7/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.7/devel/dev_dataset","sidebar":"version-1.2.7/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide","sidebar":"version-1.2.7/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.7/docusaurus","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.7/pipeline/elastic","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.7/pipeline/importer","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.7/pipeline/intro","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.7/pipeline/logstash","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.7/pipeline/nfdump","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.7/pipeline/tstat","sidebar":"version-1.2.7/Pipeline"}]},{"name":"1.2.6","label":"1.2.6","isLast":false,"path":"/netsage-pipeline/docs/1.2.6","mainDocId":"pipeline","docs":[{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.6/deploy/choose_install","sidebar":"version-1.2.6/Pipeline"},{"id":"deploy/docker_install","path":"/netsage-pipeline/docs/1.2.6/deploy/docker_install","sidebar":"version-1.2.6/Pipeline"},{"id":"deploy/install","path":"/netsage-pipeline/docs/1.2.6/deploy/install","sidebar":"version-1.2.6/Pipeline"},{"id":"devel/docker","path":"/netsage-pipeline/docs/1.2.6/devel/docker","sidebar":"version-1.2.6/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.6/docusaurus","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline","path":"/netsage-pipeline/docs/1.2.6/pipeline","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline_importer","path":"/netsage-pipeline/docs/1.2.6/pipeline_importer","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline_logstash","path":"/netsage-pipeline/docs/1.2.6/pipeline_logstash","sidebar":"version-1.2.6/Pipeline"}]},{"name":"1.2.5","label":"1.2.5","isLast":false,"path":"/netsage-pipeline/docs/1.2.5","mainDocId":"pipeline","docs":[{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.5/deploy/choose_install","sidebar":"version-1.2.5/Pipeline"},{"id":"deploy/docker_install","path":"/netsage-pipeline/docs/1.2.5/deploy/docker_install","sidebar":"version-1.2.5/Pipeline"},{"id":"deploy/install","path":"/netsage-pipeline/docs/1.2.5/deploy/install","sidebar":"version-1.2.5/Pipeline"},{"id":"devel/docker","path":"/netsage-pipeline/docs/1.2.5/devel/docker","sidebar":"version-1.2.5/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.5/docusaurus","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline","path":"/netsage-pipeline/docs/1.2.5/pipeline","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline_importer","path":"/netsage-pipeline/docs/1.2.5/pipeline_importer","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline_logstash","path":"/netsage-pipeline/docs/1.2.5/pipeline_logstash","sidebar":"version-1.2.5/Pipeline"}]}]}}}')},function(e){e.exports=JSON.parse('{"defaultLocale":"en","locales":["en"],"localeConfigs":{"en":{"label":"en","direction":"ltr"}},"currentLocale":"en"}')},function(e){e.exports=JSON.parse('{"docusaurusVersion":"2.0.0-alpha.72","siteVersion":"0.0.0","pluginVersions":{"docusaurus-plugin-content-docs":{"type":"package","name":"@docusaurus/plugin-content-docs","version":"2.0.0-alpha.72"},"docusaurus-plugin-content-blog":{"type":"package","name":"@docusaurus/plugin-content-blog","version":"2.0.0-alpha.72"},"docusaurus-plugin-content-pages":{"type":"package","name":"@docusaurus/plugin-content-pages","version":"2.0.0-alpha.72"},"docusaurus-plugin-sitemap":{"type":"package","name":"@docusaurus/plugin-sitemap","version":"2.0.0-alpha.72"},"docusaurus-theme-classic":{"type":"package","name":"@docusaurus/theme-classic","version":"2.0.0-alpha.72"}}}')},function(e,n,t){"use strict";var i=t(29),r={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},o={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},a={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},l={};function s(e){return i.isMemo(e)?a:l[e.$$typeof]||r}l[i.ForwardRef]={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},l[i.Memo]=a;var c=Object.defineProperty,u=Object.getOwnPropertyNames,p=Object.getOwnPropertySymbols,d=Object.getOwnPropertyDescriptor,f=Object.getPrototypeOf,m=Object.prototype;e.exports=function e(n,t,i){if("string"!=typeof t){if(m){var r=f(t);r&&r!==m&&e(n,r,i)}var a=u(t);p&&(a=a.concat(p(t)));for(var l=s(n),g=s(t),h=0;h<a.length;++h){var v=a[h];if(!(o[v]||i&&i[v]||g&&g[v]||l&&l[v])){var b=d(t,v);try{c(n,v,b)}catch(y){}}}}return n}},function(e,n,t){"use strict";(function(e){t.d(n,"a",(function(){return he}));var i,r,o,a,l=t(1),s=t.n(l),c=t(36),u=t.n(c),p=t(37),d=t.n(p),f=t(0),m=t.n(f),g=t(12),h=t.n(g),v="bodyAttributes",b="htmlAttributes",y="titleAttributes",k={BASE:"base",BODY:"body",HEAD:"head",HTML:"html",LINK:"link",META:"meta",NOSCRIPT:"noscript",SCRIPT:"script",STYLE:"style",TITLE:"title"},w=(Object.keys(k).map((function(e){return k[e]})),"charset"),_="cssText",x="href",E="http-equiv",S="innerHTML",T="itemprop",P="name",C="property",O="rel",A="src",R="target",N={accesskey:"accessKey",charset:"charSet",class:"className",contenteditable:"contentEditable",contextmenu:"contextMenu","http-equiv":"httpEquiv",itemprop:"itemProp",tabindex:"tabIndex"},I="defaultTitle",L="defer",D="encodeSpecialCharacters",j="onChangeClientState",M="titleTemplate",F=Object.keys(N).reduce((function(e,n){return e[N[n]]=n,e}),{}),B=[k.NOSCRIPT,k.SCRIPT,k.STYLE],z="data-react-helmet",U="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},$=function(e,n){if(!(e instanceof n))throw new TypeError("Cannot call a class as a function")},G=function(){function e(e,n){for(var t=0;t<n.length;t++){var i=n[t];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(e,i.key,i)}}return function(n,t,i){return t&&e(n.prototype,t),i&&e(n,i),n}}(),q=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e},H=function(e,n){var t={};for(var i in e)n.indexOf(i)>=0||Object.prototype.hasOwnProperty.call(e,i)&&(t[i]=e[i]);return t},W=function(e,n){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!n||"object"!=typeof n&&"function"!=typeof n?e:n},V=function(e){var n=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];return!1===n?String(e):String(e).replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")},K=function(e){var n=Z(e,k.TITLE),t=Z(e,M);if(t&&n)return t.replace(/%s/g,(function(){return Array.isArray(n)?n.join(""):n}));var i=Z(e,I);return n||i||void 0},Y=function(e){return Z(e,j)||function(){}},Q=function(e,n){return n.filter((function(n){return void 0!==n[e]})).map((function(n){return n[e]})).reduce((function(e,n){return q({},e,n)}),{})},X=function(e,n){return n.filter((function(e){return void 0!==e[k.BASE]})).map((function(e){return e[k.BASE]})).reverse().reduce((function(n,t){if(!n.length)for(var i=Object.keys(t),r=0;r<i.length;r++){var o=i[r].toLowerCase();if(-1!==e.indexOf(o)&&t[o])return n.concat(t)}return n}),[])},J=function(e,n,t){var i={};return t.filter((function(n){return!!Array.isArray(n[e])||(void 0!==n[e]&&re("Helmet: "+e+' should be of type "Array". Instead found type "'+U(n[e])+'"'),!1)})).map((function(n){return n[e]})).reverse().reduce((function(e,t){var r={};t.filter((function(e){for(var t=void 0,o=Object.keys(e),a=0;a<o.length;a++){var l=o[a],s=l.toLowerCase();-1===n.indexOf(s)||t===O&&"canonical"===e[t].toLowerCase()||s===O&&"stylesheet"===e[s].toLowerCase()||(t=s),-1===n.indexOf(l)||l!==S&&l!==_&&l!==T||(t=l)}if(!t||!e[t])return!1;var c=e[t].toLowerCase();return i[t]||(i[t]={}),r[t]||(r[t]={}),!i[t][c]&&(r[t][c]=!0,!0)})).reverse().forEach((function(n){return e.push(n)}));for(var o=Object.keys(r),a=0;a<o.length;a++){var l=o[a],s=h()({},i[l],r[l]);i[l]=s}return e}),[]).reverse()},Z=function(e,n){for(var t=e.length-1;t>=0;t--){var i=e[t];if(i.hasOwnProperty(n))return i[n]}return null},ee=(i=Date.now(),function(e){var n=Date.now();n-i>16?(i=n,e(n)):setTimeout((function(){ee(e)}),0)}),ne=function(e){return clearTimeout(e)},te="undefined"!=typeof window?window.requestAnimationFrame&&window.requestAnimationFrame.bind(window)||window.webkitRequestAnimationFrame||window.mozRequestAnimationFrame||ee:e.requestAnimationFrame||ee,ie="undefined"!=typeof window?window.cancelAnimationFrame||window.webkitCancelAnimationFrame||window.mozCancelAnimationFrame||ne:e.cancelAnimationFrame||ne,re=function(e){return console&&"function"==typeof console.warn&&console.warn(e)},oe=null,ae=function(e,n){var t=e.baseTag,i=e.bodyAttributes,r=e.htmlAttributes,o=e.linkTags,a=e.metaTags,l=e.noscriptTags,s=e.onChangeClientState,c=e.scriptTags,u=e.styleTags,p=e.title,d=e.titleAttributes;ce(k.BODY,i),ce(k.HTML,r),se(p,d);var f={baseTag:ue(k.BASE,t),linkTags:ue(k.LINK,o),metaTags:ue(k.META,a),noscriptTags:ue(k.NOSCRIPT,l),scriptTags:ue(k.SCRIPT,c),styleTags:ue(k.STYLE,u)},m={},g={};Object.keys(f).forEach((function(e){var n=f[e],t=n.newTags,i=n.oldTags;t.length&&(m[e]=t),i.length&&(g[e]=f[e].oldTags)})),n&&n(),s(e,m,g)},le=function(e){return Array.isArray(e)?e.join(""):e},se=function(e,n){void 0!==e&&document.title!==e&&(document.title=le(e)),ce(k.TITLE,n)},ce=function(e,n){var t=document.getElementsByTagName(e)[0];if(t){for(var i=t.getAttribute(z),r=i?i.split(","):[],o=[].concat(r),a=Object.keys(n),l=0;l<a.length;l++){var s=a[l],c=n[s]||"";t.getAttribute(s)!==c&&t.setAttribute(s,c),-1===r.indexOf(s)&&r.push(s);var u=o.indexOf(s);-1!==u&&o.splice(u,1)}for(var p=o.length-1;p>=0;p--)t.removeAttribute(o[p]);r.length===o.length?t.removeAttribute(z):t.getAttribute(z)!==a.join(",")&&t.setAttribute(z,a.join(","))}},ue=function(e,n){var t=document.head||document.querySelector(k.HEAD),i=t.querySelectorAll(e+"["+"data-react-helmet]"),r=Array.prototype.slice.call(i),o=[],a=void 0;return n&&n.length&&n.forEach((function(n){var t=document.createElement(e);for(var i in n)if(n.hasOwnProperty(i))if(i===S)t.innerHTML=n.innerHTML;else if(i===_)t.styleSheet?t.styleSheet.cssText=n.cssText:t.appendChild(document.createTextNode(n.cssText));else{var l=void 0===n[i]?"":n[i];t.setAttribute(i,l)}t.setAttribute(z,"true"),r.some((function(e,n){return a=n,t.isEqualNode(e)}))?r.splice(a,1):o.push(t)})),r.forEach((function(e){return e.parentNode.removeChild(e)})),o.forEach((function(e){return t.appendChild(e)})),{oldTags:r,newTags:o}},pe=function(e){return Object.keys(e).reduce((function(n,t){var i=void 0!==e[t]?t+'="'+e[t]+'"':""+t;return n?n+" "+i:i}),"")},de=function(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return Object.keys(e).reduce((function(n,t){return n[N[t]||t]=e[t],n}),n)},fe=function(e,n,t){switch(e){case k.TITLE:return{toComponent:function(){return e=n.title,t=n.titleAttributes,(i={key:e})[z]=!0,r=de(t,i),[m.a.createElement(k.TITLE,r,e)];var e,t,i,r},toString:function(){return function(e,n,t,i){var r=pe(t),o=le(n);return r?"<"+e+' data-react-helmet="true" '+r+">"+V(o,i)+"</"+e+">":"<"+e+' data-react-helmet="true">'+V(o,i)+"</"+e+">"}(e,n.title,n.titleAttributes,t)}};case v:case b:return{toComponent:function(){return de(n)},toString:function(){return pe(n)}};default:return{toComponent:function(){return function(e,n){return n.map((function(n,t){var i,r=((i={key:t})[z]=!0,i);return Object.keys(n).forEach((function(e){var t=N[e]||e;if(t===S||t===_){var i=n.innerHTML||n.cssText;r.dangerouslySetInnerHTML={__html:i}}else r[t]=n[e]})),m.a.createElement(e,r)}))}(e,n)},toString:function(){return function(e,n,t){return n.reduce((function(n,i){var r=Object.keys(i).filter((function(e){return!(e===S||e===_)})).reduce((function(e,n){var r=void 0===i[n]?n:n+'="'+V(i[n],t)+'"';return e?e+" "+r:r}),""),o=i.innerHTML||i.cssText||"",a=-1===B.indexOf(e);return n+"<"+e+' data-react-helmet="true" '+r+(a?"/>":">"+o+"</"+e+">")}),"")}(e,n,t)}}}},me=function(e){var n=e.baseTag,t=e.bodyAttributes,i=e.encode,r=e.htmlAttributes,o=e.linkTags,a=e.metaTags,l=e.noscriptTags,s=e.scriptTags,c=e.styleTags,u=e.title,p=void 0===u?"":u,d=e.titleAttributes;return{base:fe(k.BASE,n,i),bodyAttributes:fe(v,t,i),htmlAttributes:fe(b,r,i),link:fe(k.LINK,o,i),meta:fe(k.META,a,i),noscript:fe(k.NOSCRIPT,l,i),script:fe(k.SCRIPT,s,i),style:fe(k.STYLE,c,i),title:fe(k.TITLE,{title:p,titleAttributes:d},i)}},ge=u()((function(e){return{baseTag:X([x,R],e),bodyAttributes:Q(v,e),defer:Z(e,L),encode:Z(e,D),htmlAttributes:Q(b,e),linkTags:J(k.LINK,[O,x],e),metaTags:J(k.META,[P,w,E,C,T],e),noscriptTags:J(k.NOSCRIPT,[S],e),onChangeClientState:Y(e),scriptTags:J(k.SCRIPT,[A,S],e),styleTags:J(k.STYLE,[_],e),title:K(e),titleAttributes:Q(y,e)}}),(function(e){oe&&ie(oe),e.defer?oe=te((function(){ae(e,(function(){oe=null}))})):(ae(e),oe=null)}),me)((function(){return null})),he=(r=ge,a=o=function(e){function n(){return $(this,n),W(this,e.apply(this,arguments))}return function(e,n){if("function"!=typeof n&&null!==n)throw new TypeError("Super expression must either be null or a function, not "+typeof n);e.prototype=Object.create(n&&n.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),n&&(Object.setPrototypeOf?Object.setPrototypeOf(e,n):e.__proto__=n)}(n,e),n.prototype.shouldComponentUpdate=function(e){return!d()(this.props,e)},n.prototype.mapNestedChildrenToProps=function(e,n){if(!n)return null;switch(e.type){case k.SCRIPT:case k.NOSCRIPT:return{innerHTML:n};case k.STYLE:return{cssText:n}}throw new Error("<"+e.type+" /> elements are self-closing and can not contain children. Refer to our API for more information.")},n.prototype.flattenArrayTypeChildren=function(e){var n,t=e.child,i=e.arrayTypeChildren,r=e.newChildProps,o=e.nestedChildren;return q({},i,((n={})[t.type]=[].concat(i[t.type]||[],[q({},r,this.mapNestedChildrenToProps(t,o))]),n))},n.prototype.mapObjectTypeChildren=function(e){var n,t,i=e.child,r=e.newProps,o=e.newChildProps,a=e.nestedChildren;switch(i.type){case k.TITLE:return q({},r,((n={})[i.type]=a,n.titleAttributes=q({},o),n));case k.BODY:return q({},r,{bodyAttributes:q({},o)});case k.HTML:return q({},r,{htmlAttributes:q({},o)})}return q({},r,((t={})[i.type]=q({},o),t))},n.prototype.mapArrayTypeChildrenToProps=function(e,n){var t=q({},n);return Object.keys(e).forEach((function(n){var i;t=q({},t,((i={})[n]=e[n],i))})),t},n.prototype.warnOnInvalidChildren=function(e,n){return!0},n.prototype.mapChildrenToProps=function(e,n){var t=this,i={};return m.a.Children.forEach(e,(function(e){if(e&&e.props){var r=e.props,o=r.children,a=function(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return Object.keys(e).reduce((function(n,t){return n[F[t]||t]=e[t],n}),n)}(H(r,["children"]));switch(t.warnOnInvalidChildren(e,o),e.type){case k.LINK:case k.META:case k.NOSCRIPT:case k.SCRIPT:case k.STYLE:i=t.flattenArrayTypeChildren({child:e,arrayTypeChildren:i,newChildProps:a,nestedChildren:o});break;default:n=t.mapObjectTypeChildren({child:e,newProps:n,newChildProps:a,nestedChildren:o})}}})),n=this.mapArrayTypeChildrenToProps(i,n)},n.prototype.render=function(){var e=this.props,n=e.children,t=H(e,["children"]),i=q({},t);return n&&(i=this.mapChildrenToProps(n,i)),m.a.createElement(r,i)},G(n,null,[{key:"canUseDOM",set:function(e){r.canUseDOM=e}}]),n}(m.a.Component),o.propTypes={base:s.a.object,bodyAttributes:s.a.object,children:s.a.oneOfType([s.a.arrayOf(s.a.node),s.a.node]),defaultTitle:s.a.string,defer:s.a.bool,encodeSpecialCharacters:s.a.bool,htmlAttributes:s.a.object,link:s.a.arrayOf(s.a.object),meta:s.a.arrayOf(s.a.object),noscript:s.a.arrayOf(s.a.object),onChangeClientState:s.a.func,script:s.a.arrayOf(s.a.object),style:s.a.arrayOf(s.a.object),title:s.a.string,titleAttributes:s.a.object,titleTemplate:s.a.string},o.defaultProps={defer:!0,encodeSpecialCharacters:!0},o.peek=r.peek,o.rewind=function(){var e=r.rewind();return e||(e=me({baseTag:[],bodyAttributes:{},encodeSpecialCharacters:!0,htmlAttributes:{},linkTags:[],metaTags:[],noscriptTags:[],scriptTags:[],styleTags:[],title:"",titleAttributes:{}})),e},a);he.renderStatic=he.rewind}).call(this,t(28))},function(e,n,t){"use strict";var i,r=t(0),o=(i=r)&&"object"==typeof i&&"default"in i?i.default:i;function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}var l=!("undefined"==typeof window||!window.document||!window.document.createElement);e.exports=function(e,n,t){if("function"!=typeof e)throw new Error("Expected reducePropsToState to be a function.");if("function"!=typeof n)throw new Error("Expected handleStateChangeOnClient to be a function.");if(void 0!==t&&"function"!=typeof t)throw new Error("Expected mapStateOnServer to either be undefined or a function.");return function(i){if("function"!=typeof i)throw new Error("Expected WrappedComponent to be a React component.");var s,c=[];function u(){s=e(c.map((function(e){return e.props}))),p.canUseDOM?n(s):t&&(s=t(s))}var p=function(e){var n,t;function r(){return e.apply(this,arguments)||this}t=e,(n=r).prototype=Object.create(t.prototype),n.prototype.constructor=n,n.__proto__=t,r.peek=function(){return s},r.rewind=function(){if(r.canUseDOM)throw new Error("You may only call rewind() on the server. Call peek() to read the current state.");var e=s;return s=void 0,c=[],e};var a=r.prototype;return a.UNSAFE_componentWillMount=function(){c.push(this),u()},a.componentDidUpdate=function(){u()},a.componentWillUnmount=function(){var e=c.indexOf(this);c.splice(e,1),u()},a.render=function(){return o.createElement(i,this.props)},r}(r.PureComponent);return a(p,"displayName","SideEffect("+function(e){return e.displayName||e.name||"Component"}(i)+")"),a(p,"canUseDOM",l),p}}},function(e,n){var t="undefined"!=typeof Element,i="function"==typeof Map,r="function"==typeof Set,o="function"==typeof ArrayBuffer&&!!ArrayBuffer.isView;function a(e,n){if(e===n)return!0;if(e&&n&&"object"==typeof e&&"object"==typeof n){if(e.constructor!==n.constructor)return!1;var l,s,c,u;if(Array.isArray(e)){if((l=e.length)!=n.length)return!1;for(s=l;0!=s--;)if(!a(e[s],n[s]))return!1;return!0}if(i&&e instanceof Map&&n instanceof Map){if(e.size!==n.size)return!1;for(u=e.entries();!(s=u.next()).done;)if(!n.has(s.value[0]))return!1;for(u=e.entries();!(s=u.next()).done;)if(!a(s.value[1],n.get(s.value[0])))return!1;return!0}if(r&&e instanceof Set&&n instanceof Set){if(e.size!==n.size)return!1;for(u=e.entries();!(s=u.next()).done;)if(!n.has(s.value[0]))return!1;return!0}if(o&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(n)){if((l=e.length)!=n.length)return!1;for(s=l;0!=s--;)if(e[s]!==n[s])return!1;return!0}if(e.constructor===RegExp)return e.source===n.source&&e.flags===n.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===n.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===n.toString();if((l=(c=Object.keys(e)).length)!==Object.keys(n).length)return!1;for(s=l;0!=s--;)if(!Object.prototype.hasOwnProperty.call(n,c[s]))return!1;if(t&&e instanceof Element)return!1;for(s=l;0!=s--;)if(("_owner"!==c[s]&&"__v"!==c[s]&&"__o"!==c[s]||!e.$$typeof)&&!a(e[c[s]],n[c[s]]))return!1;return!0}return e!=e&&n!=n}e.exports=function(e,n){try{return a(e,n)}catch(t){if((t.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw t}}},function(e,n,t){e.exports=t(54)},function(e,n,t){"use strict";var i=t(12),r="function"==typeof Symbol&&Symbol.for,o=r?Symbol.for("react.element"):60103,a=r?Symbol.for("react.portal"):60106,l=r?Symbol.for("react.fragment"):60107,s=r?Symbol.for("react.strict_mode"):60108,c=r?Symbol.for("react.profiler"):60114,u=r?Symbol.for("react.provider"):60109,p=r?Symbol.for("react.context"):60110,d=r?Symbol.for("react.forward_ref"):60112,f=r?Symbol.for("react.suspense"):60113,m=r?Symbol.for("react.memo"):60115,g=r?Symbol.for("react.lazy"):60116,h="function"==typeof Symbol&&Symbol.iterator;function v(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t<arguments.length;t++)n+="&args[]="+encodeURIComponent(arguments[t]);return"Minified React error #"+e+"; visit "+n+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}var b={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},y={};function k(e,n,t){this.props=e,this.context=n,this.refs=y,this.updater=t||b}function w(){}function _(e,n,t){this.props=e,this.context=n,this.refs=y,this.updater=t||b}k.prototype.isReactComponent={},k.prototype.setState=function(e,n){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error(v(85));this.updater.enqueueSetState(this,e,n,"setState")},k.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},w.prototype=k.prototype;var x=_.prototype=new w;x.constructor=_,i(x,k.prototype),x.isPureReactComponent=!0;var E={current:null},S=Object.prototype.hasOwnProperty,T={key:!0,ref:!0,__self:!0,__source:!0};function P(e,n,t){var i,r={},a=null,l=null;if(null!=n)for(i in void 0!==n.ref&&(l=n.ref),void 0!==n.key&&(a=""+n.key),n)S.call(n,i)&&!T.hasOwnProperty(i)&&(r[i]=n[i]);var s=arguments.length-2;if(1===s)r.children=t;else if(1<s){for(var c=Array(s),u=0;u<s;u++)c[u]=arguments[u+2];r.children=c}if(e&&e.defaultProps)for(i in s=e.defaultProps)void 0===r[i]&&(r[i]=s[i]);return{$$typeof:o,type:e,key:a,ref:l,props:r,_owner:E.current}}function C(e){return"object"==typeof e&&null!==e&&e.$$typeof===o}var O=/\/+/g,A=[];function R(e,n,t,i){if(A.length){var r=A.pop();return r.result=e,r.keyPrefix=n,r.func=t,r.context=i,r.count=0,r}return{result:e,keyPrefix:n,func:t,context:i,count:0}}function N(e){e.result=null,e.keyPrefix=null,e.func=null,e.context=null,e.count=0,10>A.length&&A.push(e)}function I(e,n,t,i){var r=typeof e;"undefined"!==r&&"boolean"!==r||(e=null);var l=!1;if(null===e)l=!0;else switch(r){case"string":case"number":l=!0;break;case"object":switch(e.$$typeof){case o:case a:l=!0}}if(l)return t(i,e,""===n?"."+D(e,0):n),1;if(l=0,n=""===n?".":n+":",Array.isArray(e))for(var s=0;s<e.length;s++){var c=n+D(r=e[s],s);l+=I(r,c,t,i)}else if(null===e||"object"!=typeof e?c=null:c="function"==typeof(c=h&&e[h]||e["@@iterator"])?c:null,"function"==typeof c)for(e=c.call(e),s=0;!(r=e.next()).done;)l+=I(r=r.value,c=n+D(r,s++),t,i);else if("object"===r)throw t=""+e,Error(v(31,"[object Object]"===t?"object with keys {"+Object.keys(e).join(", ")+"}":t,""));return l}function L(e,n,t){return null==e?0:I(e,"",n,t)}function D(e,n){return"object"==typeof e&&null!==e&&null!=e.key?function(e){var n={"=":"=0",":":"=2"};return"$"+(""+e).replace(/[=:]/g,(function(e){return n[e]}))}(e.key):n.toString(36)}function j(e,n){e.func.call(e.context,n,e.count++)}function M(e,n,t){var i=e.result,r=e.keyPrefix;e=e.func.call(e.context,n,e.count++),Array.isArray(e)?F(e,i,t,(function(e){return e})):null!=e&&(C(e)&&(e=function(e,n){return{$$typeof:o,type:e.type,key:n,ref:e.ref,props:e.props,_owner:e._owner}}(e,r+(!e.key||n&&n.key===e.key?"":(""+e.key).replace(O,"$&/")+"/")+t)),i.push(e))}function F(e,n,t,i,r){var o="";null!=t&&(o=(""+t).replace(O,"$&/")+"/"),L(e,M,n=R(n,o,i,r)),N(n)}var B={current:null};function z(){var e=B.current;if(null===e)throw Error(v(321));return e}var U={ReactCurrentDispatcher:B,ReactCurrentBatchConfig:{suspense:null},ReactCurrentOwner:E,IsSomeRendererActing:{current:!1},assign:i};n.Children={map:function(e,n,t){if(null==e)return e;var i=[];return F(e,i,null,n,t),i},forEach:function(e,n,t){if(null==e)return e;L(e,j,n=R(null,null,n,t)),N(n)},count:function(e){return L(e,(function(){return null}),null)},toArray:function(e){var n=[];return F(e,n,null,(function(e){return e})),n},only:function(e){if(!C(e))throw Error(v(143));return e}},n.Component=k,n.Fragment=l,n.Profiler=c,n.PureComponent=_,n.StrictMode=s,n.Suspense=f,n.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=U,n.cloneElement=function(e,n,t){if(null==e)throw Error(v(267,e));var r=i({},e.props),a=e.key,l=e.ref,s=e._owner;if(null!=n){if(void 0!==n.ref&&(l=n.ref,s=E.current),void 0!==n.key&&(a=""+n.key),e.type&&e.type.defaultProps)var c=e.type.defaultProps;for(u in n)S.call(n,u)&&!T.hasOwnProperty(u)&&(r[u]=void 0===n[u]&&void 0!==c?c[u]:n[u])}var u=arguments.length-2;if(1===u)r.children=t;else if(1<u){c=Array(u);for(var p=0;p<u;p++)c[p]=arguments[p+2];r.children=c}return{$$typeof:o,type:e.type,key:a,ref:l,props:r,_owner:s}},n.createContext=function(e,n){return void 0===n&&(n=null),(e={$$typeof:p,_calculateChangedBits:n,_currentValue:e,_currentValue2:e,_threadCount:0,Provider:null,Consumer:null}).Provider={$$typeof:u,_context:e},e.Consumer=e},n.createElement=P,n.createFactory=function(e){var n=P.bind(null,e);return n.type=e,n},n.createRef=function(){return{current:null}},n.forwardRef=function(e){return{$$typeof:d,render:e}},n.isValidElement=C,n.lazy=function(e){return{$$typeof:g,_ctor:e,_status:-1,_result:null}},n.memo=function(e,n){return{$$typeof:m,type:e,compare:void 0===n?null:n}},n.useCallback=function(e,n){return z().useCallback(e,n)},n.useContext=function(e,n){return z().useContext(e,n)},n.useDebugValue=function(){},n.useEffect=function(e,n){return z().useEffect(e,n)},n.useImperativeHandle=function(e,n,t){return z().useImperativeHandle(e,n,t)},n.useLayoutEffect=function(e,n){return z().useLayoutEffect(e,n)},n.useMemo=function(e,n){return z().useMemo(e,n)},n.useReducer=function(e,n,t){return z().useReducer(e,n,t)},n.useRef=function(e){return z().useRef(e)},n.useState=function(e){return z().useState(e)},n.version="16.14.0"},function(e,n,t){"use strict";var i=t(0),r=t(12),o=t(41);function a(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t<arguments.length;t++)n+="&args[]="+encodeURIComponent(arguments[t]);return"Minified React error #"+e+"; visit "+n+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}if(!i)throw Error(a(227));function l(e,n,t,i,r,o,a,l,s){var c=Array.prototype.slice.call(arguments,3);try{n.apply(t,c)}catch(u){this.onError(u)}}var s=!1,c=null,u=!1,p=null,d={onError:function(e){s=!0,c=e}};function f(e,n,t,i,r,o,a,u,p){s=!1,c=null,l.apply(d,arguments)}var m=null,g=null,h=null;function v(e,n,t){var i=e.type||"unknown-event";e.currentTarget=h(t),function(e,n,t,i,r,o,l,d,m){if(f.apply(this,arguments),s){if(!s)throw Error(a(198));var g=c;s=!1,c=null,u||(u=!0,p=g)}}(i,n,void 0,e),e.currentTarget=null}var b=null,y={};function k(){if(b)for(var e in y){var n=y[e],t=b.indexOf(e);if(!(-1<t))throw Error(a(96,e));if(!_[t]){if(!n.extractEvents)throw Error(a(97,e));for(var i in _[t]=n,t=n.eventTypes){var r=void 0,o=t[i],l=n,s=i;if(x.hasOwnProperty(s))throw Error(a(99,s));x[s]=o;var c=o.phasedRegistrationNames;if(c){for(r in c)c.hasOwnProperty(r)&&w(c[r],l,s);r=!0}else o.registrationName?(w(o.registrationName,l,s),r=!0):r=!1;if(!r)throw Error(a(98,i,e))}}}}function w(e,n,t){if(E[e])throw Error(a(100,e));E[e]=n,S[e]=n.eventTypes[t].dependencies}var _=[],x={},E={},S={};function T(e){var n,t=!1;for(n in e)if(e.hasOwnProperty(n)){var i=e[n];if(!y.hasOwnProperty(n)||y[n]!==i){if(y[n])throw Error(a(102,n));y[n]=i,t=!0}}t&&k()}var P=!("undefined"==typeof window||void 0===window.document||void 0===window.document.createElement),C=null,O=null,A=null;function R(e){if(e=g(e)){if("function"!=typeof C)throw Error(a(280));var n=e.stateNode;n&&(n=m(n),C(e.stateNode,e.type,n))}}function N(e){O?A?A.push(e):A=[e]:O=e}function I(){if(O){var e=O,n=A;if(A=O=null,R(e),n)for(e=0;e<n.length;e++)R(n[e])}}function L(e,n){return e(n)}function D(e,n,t,i,r){return e(n,t,i,r)}function j(){}var M=L,F=!1,B=!1;function z(){null===O&&null===A||(j(),I())}function U(e,n,t){if(B)return e(n,t);B=!0;try{return M(e,n,t)}finally{B=!1,z()}}var $=/^[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD][:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-.0-9\u00B7\u0300-\u036F\u203F-\u2040]*$/,G=Object.prototype.hasOwnProperty,q={},H={};function W(e,n,t,i,r,o){this.acceptsBooleans=2===n||3===n||4===n,this.attributeName=i,this.attributeNamespace=r,this.mustUseProperty=t,this.propertyName=e,this.type=n,this.sanitizeURL=o}var V={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach((function(e){V[e]=new W(e,0,!1,e,null,!1)})),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach((function(e){var n=e[0];V[n]=new W(n,1,!1,e[1],null,!1)})),["contentEditable","draggable","spellCheck","value"].forEach((function(e){V[e]=new W(e,2,!1,e.toLowerCase(),null,!1)})),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach((function(e){V[e]=new W(e,2,!1,e,null,!1)})),"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach((function(e){V[e]=new W(e,3,!1,e.toLowerCase(),null,!1)})),["checked","multiple","muted","selected"].forEach((function(e){V[e]=new W(e,3,!0,e,null,!1)})),["capture","download"].forEach((function(e){V[e]=new W(e,4,!1,e,null,!1)})),["cols","rows","size","span"].forEach((function(e){V[e]=new W(e,6,!1,e,null,!1)})),["rowSpan","start"].forEach((function(e){V[e]=new W(e,5,!1,e.toLowerCase(),null,!1)}));var K=/[\-:]([a-z])/g;function Y(e){return e[1].toUpperCase()}"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,null,!1)})),"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,"http://www.w3.org/1999/xlink",!1)})),["xml:base","xml:lang","xml:space"].forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1)})),["tabIndex","crossOrigin"].forEach((function(e){V[e]=new W(e,1,!1,e.toLowerCase(),null,!1)})),V.xlinkHref=new W("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0),["src","href","action","formAction"].forEach((function(e){V[e]=new W(e,1,!1,e.toLowerCase(),null,!0)}));var Q=i.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED;function X(e,n,t,i){var r=V.hasOwnProperty(n)?V[n]:null;(null!==r?0===r.type:!i&&(2<n.length&&("o"===n[0]||"O"===n[0])&&("n"===n[1]||"N"===n[1])))||(function(e,n,t,i){if(null==n||function(e,n,t,i){if(null!==t&&0===t.type)return!1;switch(typeof n){case"function":case"symbol":return!0;case"boolean":return!i&&(null!==t?!t.acceptsBooleans:"data-"!==(e=e.toLowerCase().slice(0,5))&&"aria-"!==e);default:return!1}}(e,n,t,i))return!0;if(i)return!1;if(null!==t)switch(t.type){case 3:return!n;case 4:return!1===n;case 5:return isNaN(n);case 6:return isNaN(n)||1>n}return!1}(n,t,r,i)&&(t=null),i||null===r?function(e){return!!G.call(H,e)||!G.call(q,e)&&($.test(e)?H[e]=!0:(q[e]=!0,!1))}(n)&&(null===t?e.removeAttribute(n):e.setAttribute(n,""+t)):r.mustUseProperty?e[r.propertyName]=null===t?3!==r.type&&"":t:(n=r.attributeName,i=r.attributeNamespace,null===t?e.removeAttribute(n):(t=3===(r=r.type)||4===r&&!0===t?"":""+t,i?e.setAttributeNS(i,n,t):e.setAttribute(n,t))))}Q.hasOwnProperty("ReactCurrentDispatcher")||(Q.ReactCurrentDispatcher={current:null}),Q.hasOwnProperty("ReactCurrentBatchConfig")||(Q.ReactCurrentBatchConfig={suspense:null});var J=/^(.*)[\\\/]/,Z="function"==typeof Symbol&&Symbol.for,ee=Z?Symbol.for("react.element"):60103,ne=Z?Symbol.for("react.portal"):60106,te=Z?Symbol.for("react.fragment"):60107,ie=Z?Symbol.for("react.strict_mode"):60108,re=Z?Symbol.for("react.profiler"):60114,oe=Z?Symbol.for("react.provider"):60109,ae=Z?Symbol.for("react.context"):60110,le=Z?Symbol.for("react.concurrent_mode"):60111,se=Z?Symbol.for("react.forward_ref"):60112,ce=Z?Symbol.for("react.suspense"):60113,ue=Z?Symbol.for("react.suspense_list"):60120,pe=Z?Symbol.for("react.memo"):60115,de=Z?Symbol.for("react.lazy"):60116,fe=Z?Symbol.for("react.block"):60121,me="function"==typeof Symbol&&Symbol.iterator;function ge(e){return null===e||"object"!=typeof e?null:"function"==typeof(e=me&&e[me]||e["@@iterator"])?e:null}function he(e){if(null==e)return null;if("function"==typeof e)return e.displayName||e.name||null;if("string"==typeof e)return e;switch(e){case te:return"Fragment";case ne:return"Portal";case re:return"Profiler";case ie:return"StrictMode";case ce:return"Suspense";case ue:return"SuspenseList"}if("object"==typeof e)switch(e.$$typeof){case ae:return"Context.Consumer";case oe:return"Context.Provider";case se:var n=e.render;return n=n.displayName||n.name||"",e.displayName||(""!==n?"ForwardRef("+n+")":"ForwardRef");case pe:return he(e.type);case fe:return he(e.render);case de:if(e=1===e._status?e._result:null)return he(e)}return null}function ve(e){var n="";do{e:switch(e.tag){case 3:case 4:case 6:case 7:case 10:case 9:var t="";break e;default:var i=e._debugOwner,r=e._debugSource,o=he(e.type);t=null,i&&(t=he(i.type)),i=o,o="",r?o=" (at "+r.fileName.replace(J,"")+":"+r.lineNumber+")":t&&(o=" (created by "+t+")"),t="\n    in "+(i||"Unknown")+o}n+=t,e=e.return}while(e);return n}function be(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}function ye(e){var n=e.type;return(e=e.nodeName)&&"input"===e.toLowerCase()&&("checkbox"===n||"radio"===n)}function ke(e){e._valueTracker||(e._valueTracker=function(e){var n=ye(e)?"checked":"value",t=Object.getOwnPropertyDescriptor(e.constructor.prototype,n),i=""+e[n];if(!e.hasOwnProperty(n)&&void 0!==t&&"function"==typeof t.get&&"function"==typeof t.set){var r=t.get,o=t.set;return Object.defineProperty(e,n,{configurable:!0,get:function(){return r.call(this)},set:function(e){i=""+e,o.call(this,e)}}),Object.defineProperty(e,n,{enumerable:t.enumerable}),{getValue:function(){return i},setValue:function(e){i=""+e},stopTracking:function(){e._valueTracker=null,delete e[n]}}}}(e))}function we(e){if(!e)return!1;var n=e._valueTracker;if(!n)return!0;var t=n.getValue(),i="";return e&&(i=ye(e)?e.checked?"true":"false":e.value),(e=i)!==t&&(n.setValue(e),!0)}function _e(e,n){var t=n.checked;return r({},n,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:null!=t?t:e._wrapperState.initialChecked})}function xe(e,n){var t=null==n.defaultValue?"":n.defaultValue,i=null!=n.checked?n.checked:n.defaultChecked;t=be(null!=n.value?n.value:t),e._wrapperState={initialChecked:i,initialValue:t,controlled:"checkbox"===n.type||"radio"===n.type?null!=n.checked:null!=n.value}}function Ee(e,n){null!=(n=n.checked)&&X(e,"checked",n,!1)}function Se(e,n){Ee(e,n);var t=be(n.value),i=n.type;if(null!=t)"number"===i?(0===t&&""===e.value||e.value!=t)&&(e.value=""+t):e.value!==""+t&&(e.value=""+t);else if("submit"===i||"reset"===i)return void e.removeAttribute("value");n.hasOwnProperty("value")?Pe(e,n.type,t):n.hasOwnProperty("defaultValue")&&Pe(e,n.type,be(n.defaultValue)),null==n.checked&&null!=n.defaultChecked&&(e.defaultChecked=!!n.defaultChecked)}function Te(e,n,t){if(n.hasOwnProperty("value")||n.hasOwnProperty("defaultValue")){var i=n.type;if(!("submit"!==i&&"reset"!==i||void 0!==n.value&&null!==n.value))return;n=""+e._wrapperState.initialValue,t||n===e.value||(e.value=n),e.defaultValue=n}""!==(t=e.name)&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,""!==t&&(e.name=t)}function Pe(e,n,t){"number"===n&&e.ownerDocument.activeElement===e||(null==t?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+t&&(e.defaultValue=""+t))}function Ce(e,n){return e=r({children:void 0},n),(n=function(e){var n="";return i.Children.forEach(e,(function(e){null!=e&&(n+=e)})),n}(n.children))&&(e.children=n),e}function Oe(e,n,t,i){if(e=e.options,n){n={};for(var r=0;r<t.length;r++)n["$"+t[r]]=!0;for(t=0;t<e.length;t++)r=n.hasOwnProperty("$"+e[t].value),e[t].selected!==r&&(e[t].selected=r),r&&i&&(e[t].defaultSelected=!0)}else{for(t=""+be(t),n=null,r=0;r<e.length;r++){if(e[r].value===t)return e[r].selected=!0,void(i&&(e[r].defaultSelected=!0));null!==n||e[r].disabled||(n=e[r])}null!==n&&(n.selected=!0)}}function Ae(e,n){if(null!=n.dangerouslySetInnerHTML)throw Error(a(91));return r({},n,{value:void 0,defaultValue:void 0,children:""+e._wrapperState.initialValue})}function Re(e,n){var t=n.value;if(null==t){if(t=n.children,n=n.defaultValue,null!=t){if(null!=n)throw Error(a(92));if(Array.isArray(t)){if(!(1>=t.length))throw Error(a(93));t=t[0]}n=t}null==n&&(n=""),t=n}e._wrapperState={initialValue:be(t)}}function Ne(e,n){var t=be(n.value),i=be(n.defaultValue);null!=t&&((t=""+t)!==e.value&&(e.value=t),null==n.defaultValue&&e.defaultValue!==t&&(e.defaultValue=t)),null!=i&&(e.defaultValue=""+i)}function Ie(e){var n=e.textContent;n===e._wrapperState.initialValue&&""!==n&&null!==n&&(e.value=n)}var Le="http://www.w3.org/1999/xhtml",De="http://www.w3.org/2000/svg";function je(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}function Me(e,n){return null==e||"http://www.w3.org/1999/xhtml"===e?je(n):"http://www.w3.org/2000/svg"===e&&"foreignObject"===n?"http://www.w3.org/1999/xhtml":e}var Fe,Be,ze=(Be=function(e,n){if(e.namespaceURI!==De||"innerHTML"in e)e.innerHTML=n;else{for((Fe=Fe||document.createElement("div")).innerHTML="<svg>"+n.valueOf().toString()+"</svg>",n=Fe.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;n.firstChild;)e.appendChild(n.firstChild)}},"undefined"!=typeof MSApp&&MSApp.execUnsafeLocalFunction?function(e,n,t,i){MSApp.execUnsafeLocalFunction((function(){return Be(e,n)}))}:Be);function Ue(e,n){if(n){var t=e.firstChild;if(t&&t===e.lastChild&&3===t.nodeType)return void(t.nodeValue=n)}e.textContent=n}function $e(e,n){var t={};return t[e.toLowerCase()]=n.toLowerCase(),t["Webkit"+e]="webkit"+n,t["Moz"+e]="moz"+n,t}var Ge={animationend:$e("Animation","AnimationEnd"),animationiteration:$e("Animation","AnimationIteration"),animationstart:$e("Animation","AnimationStart"),transitionend:$e("Transition","TransitionEnd")},qe={},He={};function We(e){if(qe[e])return qe[e];if(!Ge[e])return e;var n,t=Ge[e];for(n in t)if(t.hasOwnProperty(n)&&n in He)return qe[e]=t[n];return e}P&&(He=document.createElement("div").style,"AnimationEvent"in window||(delete Ge.animationend.animation,delete Ge.animationiteration.animation,delete Ge.animationstart.animation),"TransitionEvent"in window||delete Ge.transitionend.transition);var Ve=We("animationend"),Ke=We("animationiteration"),Ye=We("animationstart"),Qe=We("transitionend"),Xe="abort canplay canplaythrough durationchange emptied encrypted ended error loadeddata loadedmetadata loadstart pause play playing progress ratechange seeked seeking stalled suspend timeupdate volumechange waiting".split(" "),Je=new("function"==typeof WeakMap?WeakMap:Map);function Ze(e){var n=Je.get(e);return void 0===n&&(n=new Map,Je.set(e,n)),n}function en(e){var n=e,t=e;if(e.alternate)for(;n.return;)n=n.return;else{e=n;do{0!=(1026&(n=e).effectTag)&&(t=n.return),e=n.return}while(e)}return 3===n.tag?t:null}function nn(e){if(13===e.tag){var n=e.memoizedState;if(null===n&&(null!==(e=e.alternate)&&(n=e.memoizedState)),null!==n)return n.dehydrated}return null}function tn(e){if(en(e)!==e)throw Error(a(188))}function rn(e){if(!(e=function(e){var n=e.alternate;if(!n){if(null===(n=en(e)))throw Error(a(188));return n!==e?null:e}for(var t=e,i=n;;){var r=t.return;if(null===r)break;var o=r.alternate;if(null===o){if(null!==(i=r.return)){t=i;continue}break}if(r.child===o.child){for(o=r.child;o;){if(o===t)return tn(r),e;if(o===i)return tn(r),n;o=o.sibling}throw Error(a(188))}if(t.return!==i.return)t=r,i=o;else{for(var l=!1,s=r.child;s;){if(s===t){l=!0,t=r,i=o;break}if(s===i){l=!0,i=r,t=o;break}s=s.sibling}if(!l){for(s=o.child;s;){if(s===t){l=!0,t=o,i=r;break}if(s===i){l=!0,i=o,t=r;break}s=s.sibling}if(!l)throw Error(a(189))}}if(t.alternate!==i)throw Error(a(190))}if(3!==t.tag)throw Error(a(188));return t.stateNode.current===t?e:n}(e)))return null;for(var n=e;;){if(5===n.tag||6===n.tag)return n;if(n.child)n.child.return=n,n=n.child;else{if(n===e)break;for(;!n.sibling;){if(!n.return||n.return===e)return null;n=n.return}n.sibling.return=n.return,n=n.sibling}}return null}function on(e,n){if(null==n)throw Error(a(30));return null==e?n:Array.isArray(e)?Array.isArray(n)?(e.push.apply(e,n),e):(e.push(n),e):Array.isArray(n)?[e].concat(n):[e,n]}function an(e,n,t){Array.isArray(e)?e.forEach(n,t):e&&n.call(t,e)}var ln=null;function sn(e){if(e){var n=e._dispatchListeners,t=e._dispatchInstances;if(Array.isArray(n))for(var i=0;i<n.length&&!e.isPropagationStopped();i++)v(e,n[i],t[i]);else n&&v(e,n,t);e._dispatchListeners=null,e._dispatchInstances=null,e.isPersistent()||e.constructor.release(e)}}function cn(e){if(null!==e&&(ln=on(ln,e)),e=ln,ln=null,e){if(an(e,sn),ln)throw Error(a(95));if(u)throw e=p,u=!1,p=null,e}}function un(e){return(e=e.target||e.srcElement||window).correspondingUseElement&&(e=e.correspondingUseElement),3===e.nodeType?e.parentNode:e}function pn(e){if(!P)return!1;var n=(e="on"+e)in document;return n||((n=document.createElement("div")).setAttribute(e,"return;"),n="function"==typeof n[e]),n}var dn=[];function fn(e){e.topLevelType=null,e.nativeEvent=null,e.targetInst=null,e.ancestors.length=0,10>dn.length&&dn.push(e)}function mn(e,n,t,i){if(dn.length){var r=dn.pop();return r.topLevelType=e,r.eventSystemFlags=i,r.nativeEvent=n,r.targetInst=t,r}return{topLevelType:e,eventSystemFlags:i,nativeEvent:n,targetInst:t,ancestors:[]}}function gn(e){var n=e.targetInst,t=n;do{if(!t){e.ancestors.push(t);break}var i=t;if(3===i.tag)i=i.stateNode.containerInfo;else{for(;i.return;)i=i.return;i=3!==i.tag?null:i.stateNode.containerInfo}if(!i)break;5!==(n=t.tag)&&6!==n||e.ancestors.push(t),t=Rt(i)}while(t);for(t=0;t<e.ancestors.length;t++){n=e.ancestors[t];var r=un(e.nativeEvent);i=e.topLevelType;var o=e.nativeEvent,a=e.eventSystemFlags;0===t&&(a|=64);for(var l=null,s=0;s<_.length;s++){var c=_[s];c&&(c=c.extractEvents(i,n,o,r,a))&&(l=on(l,c))}cn(l)}}function hn(e,n,t){if(!t.has(e)){switch(e){case"scroll":Qn(n,"scroll",!0);break;case"focus":case"blur":Qn(n,"focus",!0),Qn(n,"blur",!0),t.set("blur",null),t.set("focus",null);break;case"cancel":case"close":pn(e)&&Qn(n,e,!0);break;case"invalid":case"submit":case"reset":break;default:-1===Xe.indexOf(e)&&Yn(e,n)}t.set(e,null)}}var vn,bn,yn,kn=!1,wn=[],_n=null,xn=null,En=null,Sn=new Map,Tn=new Map,Pn=[],Cn="mousedown mouseup touchcancel touchend touchstart auxclick dblclick pointercancel pointerdown pointerup dragend dragstart drop compositionend compositionstart keydown keypress keyup input textInput close cancel copy cut paste click change contextmenu reset submit".split(" "),On="focus blur dragenter dragleave mouseover mouseout pointerover pointerout gotpointercapture lostpointercapture".split(" ");function An(e,n,t,i,r){return{blockedOn:e,topLevelType:n,eventSystemFlags:32|t,nativeEvent:r,container:i}}function Rn(e,n){switch(e){case"focus":case"blur":_n=null;break;case"dragenter":case"dragleave":xn=null;break;case"mouseover":case"mouseout":En=null;break;case"pointerover":case"pointerout":Sn.delete(n.pointerId);break;case"gotpointercapture":case"lostpointercapture":Tn.delete(n.pointerId)}}function Nn(e,n,t,i,r,o){return null===e||e.nativeEvent!==o?(e=An(n,t,i,r,o),null!==n&&(null!==(n=Nt(n))&&bn(n)),e):(e.eventSystemFlags|=i,e)}function In(e){var n=Rt(e.target);if(null!==n){var t=en(n);if(null!==t)if(13===(n=t.tag)){if(null!==(n=nn(t)))return e.blockedOn=n,void o.unstable_runWithPriority(e.priority,(function(){yn(t)}))}else if(3===n&&t.stateNode.hydrate)return void(e.blockedOn=3===t.tag?t.stateNode.containerInfo:null)}e.blockedOn=null}function Ln(e){if(null!==e.blockedOn)return!1;var n=et(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);if(null!==n){var t=Nt(n);return null!==t&&bn(t),e.blockedOn=n,!1}return!0}function Dn(e,n,t){Ln(e)&&t.delete(n)}function jn(){for(kn=!1;0<wn.length;){var e=wn[0];if(null!==e.blockedOn){null!==(e=Nt(e.blockedOn))&&vn(e);break}var n=et(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);null!==n?e.blockedOn=n:wn.shift()}null!==_n&&Ln(_n)&&(_n=null),null!==xn&&Ln(xn)&&(xn=null),null!==En&&Ln(En)&&(En=null),Sn.forEach(Dn),Tn.forEach(Dn)}function Mn(e,n){e.blockedOn===n&&(e.blockedOn=null,kn||(kn=!0,o.unstable_scheduleCallback(o.unstable_NormalPriority,jn)))}function Fn(e){function n(n){return Mn(n,e)}if(0<wn.length){Mn(wn[0],e);for(var t=1;t<wn.length;t++){var i=wn[t];i.blockedOn===e&&(i.blockedOn=null)}}for(null!==_n&&Mn(_n,e),null!==xn&&Mn(xn,e),null!==En&&Mn(En,e),Sn.forEach(n),Tn.forEach(n),t=0;t<Pn.length;t++)(i=Pn[t]).blockedOn===e&&(i.blockedOn=null);for(;0<Pn.length&&null===(t=Pn[0]).blockedOn;)In(t),null===t.blockedOn&&Pn.shift()}var Bn={},zn=new Map,Un=new Map,$n=["abort","abort",Ve,"animationEnd",Ke,"animationIteration",Ye,"animationStart","canplay","canPlay","canplaythrough","canPlayThrough","durationchange","durationChange","emptied","emptied","encrypted","encrypted","ended","ended","error","error","gotpointercapture","gotPointerCapture","load","load","loadeddata","loadedData","loadedmetadata","loadedMetadata","loadstart","loadStart","lostpointercapture","lostPointerCapture","playing","playing","progress","progress","seeking","seeking","stalled","stalled","suspend","suspend","timeupdate","timeUpdate",Qe,"transitionEnd","waiting","waiting"];function Gn(e,n){for(var t=0;t<e.length;t+=2){var i=e[t],r=e[t+1],o="on"+(r[0].toUpperCase()+r.slice(1));o={phasedRegistrationNames:{bubbled:o,captured:o+"Capture"},dependencies:[i],eventPriority:n},Un.set(i,n),zn.set(i,o),Bn[r]=o}}Gn("blur blur cancel cancel click click close close contextmenu contextMenu copy copy cut cut auxclick auxClick dblclick doubleClick dragend dragEnd dragstart dragStart drop drop focus focus input input invalid invalid keydown keyDown keypress keyPress keyup keyUp mousedown mouseDown mouseup mouseUp paste paste pause pause play play pointercancel pointerCancel pointerdown pointerDown pointerup pointerUp ratechange rateChange reset reset seeked seeked submit submit touchcancel touchCancel touchend touchEnd touchstart touchStart volumechange volumeChange".split(" "),0),Gn("drag drag dragenter dragEnter dragexit dragExit dragleave dragLeave dragover dragOver mousemove mouseMove mouseout mouseOut mouseover mouseOver pointermove pointerMove pointerout pointerOut pointerover pointerOver scroll scroll toggle toggle touchmove touchMove wheel wheel".split(" "),1),Gn($n,2);for(var qn="change selectionchange textInput compositionstart compositionend compositionupdate".split(" "),Hn=0;Hn<qn.length;Hn++)Un.set(qn[Hn],0);var Wn=o.unstable_UserBlockingPriority,Vn=o.unstable_runWithPriority,Kn=!0;function Yn(e,n){Qn(n,e,!1)}function Qn(e,n,t){var i=Un.get(n);switch(void 0===i?2:i){case 0:i=Xn.bind(null,n,1,e);break;case 1:i=Jn.bind(null,n,1,e);break;default:i=Zn.bind(null,n,1,e)}t?e.addEventListener(n,i,!0):e.addEventListener(n,i,!1)}function Xn(e,n,t,i){F||j();var r=Zn,o=F;F=!0;try{D(r,e,n,t,i)}finally{(F=o)||z()}}function Jn(e,n,t,i){Vn(Wn,Zn.bind(null,e,n,t,i))}function Zn(e,n,t,i){if(Kn)if(0<wn.length&&-1<Cn.indexOf(e))e=An(null,e,n,t,i),wn.push(e);else{var r=et(e,n,t,i);if(null===r)Rn(e,i);else if(-1<Cn.indexOf(e))e=An(r,e,n,t,i),wn.push(e);else if(!function(e,n,t,i,r){switch(n){case"focus":return _n=Nn(_n,e,n,t,i,r),!0;case"dragenter":return xn=Nn(xn,e,n,t,i,r),!0;case"mouseover":return En=Nn(En,e,n,t,i,r),!0;case"pointerover":var o=r.pointerId;return Sn.set(o,Nn(Sn.get(o)||null,e,n,t,i,r)),!0;case"gotpointercapture":return o=r.pointerId,Tn.set(o,Nn(Tn.get(o)||null,e,n,t,i,r)),!0}return!1}(r,e,n,t,i)){Rn(e,i),e=mn(e,i,null,n);try{U(gn,e)}finally{fn(e)}}}}function et(e,n,t,i){if(null!==(t=Rt(t=un(i)))){var r=en(t);if(null===r)t=null;else{var o=r.tag;if(13===o){if(null!==(t=nn(r)))return t;t=null}else if(3===o){if(r.stateNode.hydrate)return 3===r.tag?r.stateNode.containerInfo:null;t=null}else r!==t&&(t=null)}}e=mn(e,i,t,n);try{U(gn,e)}finally{fn(e)}return null}var nt={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},tt=["Webkit","ms","Moz","O"];function it(e,n,t){return null==n||"boolean"==typeof n||""===n?"":t||"number"!=typeof n||0===n||nt.hasOwnProperty(e)&&nt[e]?(""+n).trim():n+"px"}function rt(e,n){for(var t in e=e.style,n)if(n.hasOwnProperty(t)){var i=0===t.indexOf("--"),r=it(t,n[t],i);"float"===t&&(t="cssFloat"),i?e.setProperty(t,r):e[t]=r}}Object.keys(nt).forEach((function(e){tt.forEach((function(n){n=n+e.charAt(0).toUpperCase()+e.substring(1),nt[n]=nt[e]}))}));var ot=r({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function at(e,n){if(n){if(ot[e]&&(null!=n.children||null!=n.dangerouslySetInnerHTML))throw Error(a(137,e,""));if(null!=n.dangerouslySetInnerHTML){if(null!=n.children)throw Error(a(60));if("object"!=typeof n.dangerouslySetInnerHTML||!("__html"in n.dangerouslySetInnerHTML))throw Error(a(61))}if(null!=n.style&&"object"!=typeof n.style)throw Error(a(62,""))}}function lt(e,n){if(-1===e.indexOf("-"))return"string"==typeof n.is;switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}var st=Le;function ct(e,n){var t=Ze(e=9===e.nodeType||11===e.nodeType?e:e.ownerDocument);n=S[n];for(var i=0;i<n.length;i++)hn(n[i],e,t)}function ut(){}function pt(e){if(void 0===(e=e||("undefined"!=typeof document?document:void 0)))return null;try{return e.activeElement||e.body}catch(n){return e.body}}function dt(e){for(;e&&e.firstChild;)e=e.firstChild;return e}function ft(e,n){var t,i=dt(e);for(e=0;i;){if(3===i.nodeType){if(t=e+i.textContent.length,e<=n&&t>=n)return{node:i,offset:n-e};e=t}e:{for(;i;){if(i.nextSibling){i=i.nextSibling;break e}i=i.parentNode}i=void 0}i=dt(i)}}function mt(e,n){return!(!e||!n)&&(e===n||(!e||3!==e.nodeType)&&(n&&3===n.nodeType?mt(e,n.parentNode):"contains"in e?e.contains(n):!!e.compareDocumentPosition&&!!(16&e.compareDocumentPosition(n))))}function gt(){for(var e=window,n=pt();n instanceof e.HTMLIFrameElement;){try{var t="string"==typeof n.contentWindow.location.href}catch(i){t=!1}if(!t)break;n=pt((e=n.contentWindow).document)}return n}function ht(e){var n=e&&e.nodeName&&e.nodeName.toLowerCase();return n&&("input"===n&&("text"===e.type||"search"===e.type||"tel"===e.type||"url"===e.type||"password"===e.type)||"textarea"===n||"true"===e.contentEditable)}var vt="$?",bt="$!",yt=null,kt=null;function wt(e,n){switch(e){case"button":case"input":case"select":case"textarea":return!!n.autoFocus}return!1}function _t(e,n){return"textarea"===e||"option"===e||"noscript"===e||"string"==typeof n.children||"number"==typeof n.children||"object"==typeof n.dangerouslySetInnerHTML&&null!==n.dangerouslySetInnerHTML&&null!=n.dangerouslySetInnerHTML.__html}var xt="function"==typeof setTimeout?setTimeout:void 0,Et="function"==typeof clearTimeout?clearTimeout:void 0;function St(e){for(;null!=e;e=e.nextSibling){var n=e.nodeType;if(1===n||3===n)break}return e}function Tt(e){e=e.previousSibling;for(var n=0;e;){if(8===e.nodeType){var t=e.data;if("$"===t||t===bt||t===vt){if(0===n)return e;n--}else"/$"===t&&n++}e=e.previousSibling}return null}var Pt=Math.random().toString(36).slice(2),Ct="__reactInternalInstance$"+Pt,Ot="__reactEventHandlers$"+Pt,At="__reactContainere$"+Pt;function Rt(e){var n=e[Ct];if(n)return n;for(var t=e.parentNode;t;){if(n=t[At]||t[Ct]){if(t=n.alternate,null!==n.child||null!==t&&null!==t.child)for(e=Tt(e);null!==e;){if(t=e[Ct])return t;e=Tt(e)}return n}t=(e=t).parentNode}return null}function Nt(e){return!(e=e[Ct]||e[At])||5!==e.tag&&6!==e.tag&&13!==e.tag&&3!==e.tag?null:e}function It(e){if(5===e.tag||6===e.tag)return e.stateNode;throw Error(a(33))}function Lt(e){return e[Ot]||null}function Dt(e){do{e=e.return}while(e&&5!==e.tag);return e||null}function jt(e,n){var t=e.stateNode;if(!t)return null;var i=m(t);if(!i)return null;t=i[n];e:switch(n){case"onClick":case"onClickCapture":case"onDoubleClick":case"onDoubleClickCapture":case"onMouseDown":case"onMouseDownCapture":case"onMouseMove":case"onMouseMoveCapture":case"onMouseUp":case"onMouseUpCapture":case"onMouseEnter":(i=!i.disabled)||(i=!("button"===(e=e.type)||"input"===e||"select"===e||"textarea"===e)),e=!i;break e;default:e=!1}if(e)return null;if(t&&"function"!=typeof t)throw Error(a(231,n,typeof t));return t}function Mt(e,n,t){(n=jt(e,t.dispatchConfig.phasedRegistrationNames[n]))&&(t._dispatchListeners=on(t._dispatchListeners,n),t._dispatchInstances=on(t._dispatchInstances,e))}function Ft(e){if(e&&e.dispatchConfig.phasedRegistrationNames){for(var n=e._targetInst,t=[];n;)t.push(n),n=Dt(n);for(n=t.length;0<n--;)Mt(t[n],"captured",e);for(n=0;n<t.length;n++)Mt(t[n],"bubbled",e)}}function Bt(e,n,t){e&&t&&t.dispatchConfig.registrationName&&(n=jt(e,t.dispatchConfig.registrationName))&&(t._dispatchListeners=on(t._dispatchListeners,n),t._dispatchInstances=on(t._dispatchInstances,e))}function zt(e){e&&e.dispatchConfig.registrationName&&Bt(e._targetInst,null,e)}function Ut(e){an(e,Ft)}var $t=null,Gt=null,qt=null;function Ht(){if(qt)return qt;var e,n,t=Gt,i=t.length,r="value"in $t?$t.value:$t.textContent,o=r.length;for(e=0;e<i&&t[e]===r[e];e++);var a=i-e;for(n=1;n<=a&&t[i-n]===r[o-n];n++);return qt=r.slice(e,1<n?1-n:void 0)}function Wt(){return!0}function Vt(){return!1}function Kt(e,n,t,i){for(var r in this.dispatchConfig=e,this._targetInst=n,this.nativeEvent=t,e=this.constructor.Interface)e.hasOwnProperty(r)&&((n=e[r])?this[r]=n(t):"target"===r?this.target=i:this[r]=t[r]);return this.isDefaultPrevented=(null!=t.defaultPrevented?t.defaultPrevented:!1===t.returnValue)?Wt:Vt,this.isPropagationStopped=Vt,this}function Yt(e,n,t,i){if(this.eventPool.length){var r=this.eventPool.pop();return this.call(r,e,n,t,i),r}return new this(e,n,t,i)}function Qt(e){if(!(e instanceof this))throw Error(a(279));e.destructor(),10>this.eventPool.length&&this.eventPool.push(e)}function Xt(e){e.eventPool=[],e.getPooled=Yt,e.release=Qt}r(Kt.prototype,{preventDefault:function(){this.defaultPrevented=!0;var e=this.nativeEvent;e&&(e.preventDefault?e.preventDefault():"unknown"!=typeof e.returnValue&&(e.returnValue=!1),this.isDefaultPrevented=Wt)},stopPropagation:function(){var e=this.nativeEvent;e&&(e.stopPropagation?e.stopPropagation():"unknown"!=typeof e.cancelBubble&&(e.cancelBubble=!0),this.isPropagationStopped=Wt)},persist:function(){this.isPersistent=Wt},isPersistent:Vt,destructor:function(){var e,n=this.constructor.Interface;for(e in n)this[e]=null;this.nativeEvent=this._targetInst=this.dispatchConfig=null,this.isPropagationStopped=this.isDefaultPrevented=Vt,this._dispatchInstances=this._dispatchListeners=null}}),Kt.Interface={type:null,target:null,currentTarget:function(){return null},eventPhase:null,bubbles:null,cancelable:null,timeStamp:function(e){return e.timeStamp||Date.now()},defaultPrevented:null,isTrusted:null},Kt.extend=function(e){function n(){}function t(){return i.apply(this,arguments)}var i=this;n.prototype=i.prototype;var o=new n;return r(o,t.prototype),t.prototype=o,t.prototype.constructor=t,t.Interface=r({},i.Interface,e),t.extend=i.extend,Xt(t),t},Xt(Kt);var Jt=Kt.extend({data:null}),Zt=Kt.extend({data:null}),ei=[9,13,27,32],ni=P&&"CompositionEvent"in window,ti=null;P&&"documentMode"in document&&(ti=document.documentMode);var ii=P&&"TextEvent"in window&&!ti,ri=P&&(!ni||ti&&8<ti&&11>=ti),oi=String.fromCharCode(32),ai={beforeInput:{phasedRegistrationNames:{bubbled:"onBeforeInput",captured:"onBeforeInputCapture"},dependencies:["compositionend","keypress","textInput","paste"]},compositionEnd:{phasedRegistrationNames:{bubbled:"onCompositionEnd",captured:"onCompositionEndCapture"},dependencies:"blur compositionend keydown keypress keyup mousedown".split(" ")},compositionStart:{phasedRegistrationNames:{bubbled:"onCompositionStart",captured:"onCompositionStartCapture"},dependencies:"blur compositionstart keydown keypress keyup mousedown".split(" ")},compositionUpdate:{phasedRegistrationNames:{bubbled:"onCompositionUpdate",captured:"onCompositionUpdateCapture"},dependencies:"blur compositionupdate keydown keypress keyup mousedown".split(" ")}},li=!1;function si(e,n){switch(e){case"keyup":return-1!==ei.indexOf(n.keyCode);case"keydown":return 229!==n.keyCode;case"keypress":case"mousedown":case"blur":return!0;default:return!1}}function ci(e){return"object"==typeof(e=e.detail)&&"data"in e?e.data:null}var ui=!1;var pi={eventTypes:ai,extractEvents:function(e,n,t,i){var r;if(ni)e:{switch(e){case"compositionstart":var o=ai.compositionStart;break e;case"compositionend":o=ai.compositionEnd;break e;case"compositionupdate":o=ai.compositionUpdate;break e}o=void 0}else ui?si(e,t)&&(o=ai.compositionEnd):"keydown"===e&&229===t.keyCode&&(o=ai.compositionStart);return o?(ri&&"ko"!==t.locale&&(ui||o!==ai.compositionStart?o===ai.compositionEnd&&ui&&(r=Ht()):(Gt="value"in($t=i)?$t.value:$t.textContent,ui=!0)),o=Jt.getPooled(o,n,t,i),r?o.data=r:null!==(r=ci(t))&&(o.data=r),Ut(o),r=o):r=null,(e=ii?function(e,n){switch(e){case"compositionend":return ci(n);case"keypress":return 32!==n.which?null:(li=!0,oi);case"textInput":return(e=n.data)===oi&&li?null:e;default:return null}}(e,t):function(e,n){if(ui)return"compositionend"===e||!ni&&si(e,n)?(e=Ht(),qt=Gt=$t=null,ui=!1,e):null;switch(e){case"paste":return null;case"keypress":if(!(n.ctrlKey||n.altKey||n.metaKey)||n.ctrlKey&&n.altKey){if(n.char&&1<n.char.length)return n.char;if(n.which)return String.fromCharCode(n.which)}return null;case"compositionend":return ri&&"ko"!==n.locale?null:n.data;default:return null}}(e,t))?((n=Zt.getPooled(ai.beforeInput,n,t,i)).data=e,Ut(n)):n=null,null===r?n:null===n?r:[r,n]}},di={color:!0,date:!0,datetime:!0,"datetime-local":!0,email:!0,month:!0,number:!0,password:!0,range:!0,search:!0,tel:!0,text:!0,time:!0,url:!0,week:!0};function fi(e){var n=e&&e.nodeName&&e.nodeName.toLowerCase();return"input"===n?!!di[e.type]:"textarea"===n}var mi={change:{phasedRegistrationNames:{bubbled:"onChange",captured:"onChangeCapture"},dependencies:"blur change click focus input keydown keyup selectionchange".split(" ")}};function gi(e,n,t){return(e=Kt.getPooled(mi.change,e,n,t)).type="change",N(t),Ut(e),e}var hi=null,vi=null;function bi(e){cn(e)}function yi(e){if(we(It(e)))return e}function ki(e,n){if("change"===e)return n}var wi=!1;function _i(){hi&&(hi.detachEvent("onpropertychange",xi),vi=hi=null)}function xi(e){if("value"===e.propertyName&&yi(vi))if(e=gi(vi,e,un(e)),F)cn(e);else{F=!0;try{L(bi,e)}finally{F=!1,z()}}}function Ei(e,n,t){"focus"===e?(_i(),vi=t,(hi=n).attachEvent("onpropertychange",xi)):"blur"===e&&_i()}function Si(e){if("selectionchange"===e||"keyup"===e||"keydown"===e)return yi(vi)}function Ti(e,n){if("click"===e)return yi(n)}function Pi(e,n){if("input"===e||"change"===e)return yi(n)}P&&(wi=pn("input")&&(!document.documentMode||9<document.documentMode));var Ci={eventTypes:mi,_isInputEventSupported:wi,extractEvents:function(e,n,t,i){var r=n?It(n):window,o=r.nodeName&&r.nodeName.toLowerCase();if("select"===o||"input"===o&&"file"===r.type)var a=ki;else if(fi(r))if(wi)a=Pi;else{a=Si;var l=Ei}else(o=r.nodeName)&&"input"===o.toLowerCase()&&("checkbox"===r.type||"radio"===r.type)&&(a=Ti);if(a&&(a=a(e,n)))return gi(a,t,i);l&&l(e,r,n),"blur"===e&&(e=r._wrapperState)&&e.controlled&&"number"===r.type&&Pe(r,"number",r.value)}},Oi=Kt.extend({view:null,detail:null}),Ai={Alt:"altKey",Control:"ctrlKey",Meta:"metaKey",Shift:"shiftKey"};function Ri(e){var n=this.nativeEvent;return n.getModifierState?n.getModifierState(e):!!(e=Ai[e])&&!!n[e]}function Ni(){return Ri}var Ii=0,Li=0,Di=!1,ji=!1,Mi=Oi.extend({screenX:null,screenY:null,clientX:null,clientY:null,pageX:null,pageY:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,getModifierState:Ni,button:null,buttons:null,relatedTarget:function(e){return e.relatedTarget||(e.fromElement===e.srcElement?e.toElement:e.fromElement)},movementX:function(e){if("movementX"in e)return e.movementX;var n=Ii;return Ii=e.screenX,Di?"mousemove"===e.type?e.screenX-n:0:(Di=!0,0)},movementY:function(e){if("movementY"in e)return e.movementY;var n=Li;return Li=e.screenY,ji?"mousemove"===e.type?e.screenY-n:0:(ji=!0,0)}}),Fi=Mi.extend({pointerId:null,width:null,height:null,pressure:null,tangentialPressure:null,tiltX:null,tiltY:null,twist:null,pointerType:null,isPrimary:null}),Bi={mouseEnter:{registrationName:"onMouseEnter",dependencies:["mouseout","mouseover"]},mouseLeave:{registrationName:"onMouseLeave",dependencies:["mouseout","mouseover"]},pointerEnter:{registrationName:"onPointerEnter",dependencies:["pointerout","pointerover"]},pointerLeave:{registrationName:"onPointerLeave",dependencies:["pointerout","pointerover"]}},zi={eventTypes:Bi,extractEvents:function(e,n,t,i,r){var o="mouseover"===e||"pointerover"===e,a="mouseout"===e||"pointerout"===e;if(o&&0==(32&r)&&(t.relatedTarget||t.fromElement)||!a&&!o)return null;(o=i.window===i?i:(o=i.ownerDocument)?o.defaultView||o.parentWindow:window,a)?(a=n,null!==(n=(n=t.relatedTarget||t.toElement)?Rt(n):null)&&(n!==en(n)||5!==n.tag&&6!==n.tag)&&(n=null)):a=null;if(a===n)return null;if("mouseout"===e||"mouseover"===e)var l=Mi,s=Bi.mouseLeave,c=Bi.mouseEnter,u="mouse";else"pointerout"!==e&&"pointerover"!==e||(l=Fi,s=Bi.pointerLeave,c=Bi.pointerEnter,u="pointer");if(e=null==a?o:It(a),o=null==n?o:It(n),(s=l.getPooled(s,a,t,i)).type=u+"leave",s.target=e,s.relatedTarget=o,(t=l.getPooled(c,n,t,i)).type=u+"enter",t.target=o,t.relatedTarget=e,u=n,(i=a)&&u)e:{for(c=u,a=0,e=l=i;e;e=Dt(e))a++;for(e=0,n=c;n;n=Dt(n))e++;for(;0<a-e;)l=Dt(l),a--;for(;0<e-a;)c=Dt(c),e--;for(;a--;){if(l===c||l===c.alternate)break e;l=Dt(l),c=Dt(c)}l=null}else l=null;for(c=l,l=[];i&&i!==c&&(null===(a=i.alternate)||a!==c);)l.push(i),i=Dt(i);for(i=[];u&&u!==c&&(null===(a=u.alternate)||a!==c);)i.push(u),u=Dt(u);for(u=0;u<l.length;u++)Bt(l[u],"bubbled",s);for(u=i.length;0<u--;)Bt(i[u],"captured",t);return 0==(64&r)?[s]:[s,t]}};var Ui="function"==typeof Object.is?Object.is:function(e,n){return e===n&&(0!==e||1/e==1/n)||e!=e&&n!=n},$i=Object.prototype.hasOwnProperty;function Gi(e,n){if(Ui(e,n))return!0;if("object"!=typeof e||null===e||"object"!=typeof n||null===n)return!1;var t=Object.keys(e),i=Object.keys(n);if(t.length!==i.length)return!1;for(i=0;i<t.length;i++)if(!$i.call(n,t[i])||!Ui(e[t[i]],n[t[i]]))return!1;return!0}var qi=P&&"documentMode"in document&&11>=document.documentMode,Hi={select:{phasedRegistrationNames:{bubbled:"onSelect",captured:"onSelectCapture"},dependencies:"blur contextmenu dragend focus keydown keyup mousedown mouseup selectionchange".split(" ")}},Wi=null,Vi=null,Ki=null,Yi=!1;function Qi(e,n){var t=n.window===n?n.document:9===n.nodeType?n:n.ownerDocument;return Yi||null==Wi||Wi!==pt(t)?null:("selectionStart"in(t=Wi)&&ht(t)?t={start:t.selectionStart,end:t.selectionEnd}:t={anchorNode:(t=(t.ownerDocument&&t.ownerDocument.defaultView||window).getSelection()).anchorNode,anchorOffset:t.anchorOffset,focusNode:t.focusNode,focusOffset:t.focusOffset},Ki&&Gi(Ki,t)?null:(Ki=t,(e=Kt.getPooled(Hi.select,Vi,e,n)).type="select",e.target=Wi,Ut(e),e))}var Xi={eventTypes:Hi,extractEvents:function(e,n,t,i,r,o){if(!(o=!(r=o||(i.window===i?i.document:9===i.nodeType?i:i.ownerDocument)))){e:{r=Ze(r),o=S.onSelect;for(var a=0;a<o.length;a++)if(!r.has(o[a])){r=!1;break e}r=!0}o=!r}if(o)return null;switch(r=n?It(n):window,e){case"focus":(fi(r)||"true"===r.contentEditable)&&(Wi=r,Vi=n,Ki=null);break;case"blur":Ki=Vi=Wi=null;break;case"mousedown":Yi=!0;break;case"contextmenu":case"mouseup":case"dragend":return Yi=!1,Qi(t,i);case"selectionchange":if(qi)break;case"keydown":case"keyup":return Qi(t,i)}return null}},Ji=Kt.extend({animationName:null,elapsedTime:null,pseudoElement:null}),Zi=Kt.extend({clipboardData:function(e){return"clipboardData"in e?e.clipboardData:window.clipboardData}}),er=Oi.extend({relatedTarget:null});function nr(e){var n=e.keyCode;return"charCode"in e?0===(e=e.charCode)&&13===n&&(e=13):e=n,10===e&&(e=13),32<=e||13===e?e:0}var tr={Esc:"Escape",Spacebar:" ",Left:"ArrowLeft",Up:"ArrowUp",Right:"ArrowRight",Down:"ArrowDown",Del:"Delete",Win:"OS",Menu:"ContextMenu",Apps:"ContextMenu",Scroll:"ScrollLock",MozPrintableKey:"Unidentified"},ir={8:"Backspace",9:"Tab",12:"Clear",13:"Enter",16:"Shift",17:"Control",18:"Alt",19:"Pause",20:"CapsLock",27:"Escape",32:" ",33:"PageUp",34:"PageDown",35:"End",36:"Home",37:"ArrowLeft",38:"ArrowUp",39:"ArrowRight",40:"ArrowDown",45:"Insert",46:"Delete",112:"F1",113:"F2",114:"F3",115:"F4",116:"F5",117:"F6",118:"F7",119:"F8",120:"F9",121:"F10",122:"F11",123:"F12",144:"NumLock",145:"ScrollLock",224:"Meta"},rr=Oi.extend({key:function(e){if(e.key){var n=tr[e.key]||e.key;if("Unidentified"!==n)return n}return"keypress"===e.type?13===(e=nr(e))?"Enter":String.fromCharCode(e):"keydown"===e.type||"keyup"===e.type?ir[e.keyCode]||"Unidentified":""},location:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,repeat:null,locale:null,getModifierState:Ni,charCode:function(e){return"keypress"===e.type?nr(e):0},keyCode:function(e){return"keydown"===e.type||"keyup"===e.type?e.keyCode:0},which:function(e){return"keypress"===e.type?nr(e):"keydown"===e.type||"keyup"===e.type?e.keyCode:0}}),or=Mi.extend({dataTransfer:null}),ar=Oi.extend({touches:null,targetTouches:null,changedTouches:null,altKey:null,metaKey:null,ctrlKey:null,shiftKey:null,getModifierState:Ni}),lr=Kt.extend({propertyName:null,elapsedTime:null,pseudoElement:null}),sr=Mi.extend({deltaX:function(e){return"deltaX"in e?e.deltaX:"wheelDeltaX"in e?-e.wheelDeltaX:0},deltaY:function(e){return"deltaY"in e?e.deltaY:"wheelDeltaY"in e?-e.wheelDeltaY:"wheelDelta"in e?-e.wheelDelta:0},deltaZ:null,deltaMode:null}),cr={eventTypes:Bn,extractEvents:function(e,n,t,i){var r=zn.get(e);if(!r)return null;switch(e){case"keypress":if(0===nr(t))return null;case"keydown":case"keyup":e=rr;break;case"blur":case"focus":e=er;break;case"click":if(2===t.button)return null;case"auxclick":case"dblclick":case"mousedown":case"mousemove":case"mouseup":case"mouseout":case"mouseover":case"contextmenu":e=Mi;break;case"drag":case"dragend":case"dragenter":case"dragexit":case"dragleave":case"dragover":case"dragstart":case"drop":e=or;break;case"touchcancel":case"touchend":case"touchmove":case"touchstart":e=ar;break;case Ve:case Ke:case Ye:e=Ji;break;case Qe:e=lr;break;case"scroll":e=Oi;break;case"wheel":e=sr;break;case"copy":case"cut":case"paste":e=Zi;break;case"gotpointercapture":case"lostpointercapture":case"pointercancel":case"pointerdown":case"pointermove":case"pointerout":case"pointerover":case"pointerup":e=Fi;break;default:e=Kt}return Ut(n=e.getPooled(r,n,t,i)),n}};if(b)throw Error(a(101));b=Array.prototype.slice.call("ResponderEventPlugin SimpleEventPlugin EnterLeaveEventPlugin ChangeEventPlugin SelectEventPlugin BeforeInputEventPlugin".split(" ")),k(),m=Lt,g=Nt,h=It,T({SimpleEventPlugin:cr,EnterLeaveEventPlugin:zi,ChangeEventPlugin:Ci,SelectEventPlugin:Xi,BeforeInputEventPlugin:pi});var ur=[],pr=-1;function dr(e){0>pr||(e.current=ur[pr],ur[pr]=null,pr--)}function fr(e,n){pr++,ur[pr]=e.current,e.current=n}var mr={},gr={current:mr},hr={current:!1},vr=mr;function br(e,n){var t=e.type.contextTypes;if(!t)return mr;var i=e.stateNode;if(i&&i.__reactInternalMemoizedUnmaskedChildContext===n)return i.__reactInternalMemoizedMaskedChildContext;var r,o={};for(r in t)o[r]=n[r];return i&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=n,e.__reactInternalMemoizedMaskedChildContext=o),o}function yr(e){return null!=(e=e.childContextTypes)}function kr(){dr(hr),dr(gr)}function wr(e,n,t){if(gr.current!==mr)throw Error(a(168));fr(gr,n),fr(hr,t)}function _r(e,n,t){var i=e.stateNode;if(e=n.childContextTypes,"function"!=typeof i.getChildContext)return t;for(var o in i=i.getChildContext())if(!(o in e))throw Error(a(108,he(n)||"Unknown",o));return r({},t,{},i)}function xr(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||mr,vr=gr.current,fr(gr,e),fr(hr,hr.current),!0}function Er(e,n,t){var i=e.stateNode;if(!i)throw Error(a(169));t?(e=_r(e,n,vr),i.__reactInternalMemoizedMergedChildContext=e,dr(hr),dr(gr),fr(gr,e)):dr(hr),fr(hr,t)}var Sr=o.unstable_runWithPriority,Tr=o.unstable_scheduleCallback,Pr=o.unstable_cancelCallback,Cr=o.unstable_requestPaint,Or=o.unstable_now,Ar=o.unstable_getCurrentPriorityLevel,Rr=o.unstable_ImmediatePriority,Nr=o.unstable_UserBlockingPriority,Ir=o.unstable_NormalPriority,Lr=o.unstable_LowPriority,Dr=o.unstable_IdlePriority,jr={},Mr=o.unstable_shouldYield,Fr=void 0!==Cr?Cr:function(){},Br=null,zr=null,Ur=!1,$r=Or(),Gr=1e4>$r?Or:function(){return Or()-$r};function qr(){switch(Ar()){case Rr:return 99;case Nr:return 98;case Ir:return 97;case Lr:return 96;case Dr:return 95;default:throw Error(a(332))}}function Hr(e){switch(e){case 99:return Rr;case 98:return Nr;case 97:return Ir;case 96:return Lr;case 95:return Dr;default:throw Error(a(332))}}function Wr(e,n){return e=Hr(e),Sr(e,n)}function Vr(e,n,t){return e=Hr(e),Tr(e,n,t)}function Kr(e){return null===Br?(Br=[e],zr=Tr(Rr,Qr)):Br.push(e),jr}function Yr(){if(null!==zr){var e=zr;zr=null,Pr(e)}Qr()}function Qr(){if(!Ur&&null!==Br){Ur=!0;var e=0;try{var n=Br;Wr(99,(function(){for(;e<n.length;e++){var t=n[e];do{t=t(!0)}while(null!==t)}})),Br=null}catch(t){throw null!==Br&&(Br=Br.slice(e+1)),Tr(Rr,Yr),t}finally{Ur=!1}}}function Xr(e,n,t){return 1073741821-(1+((1073741821-e+n/10)/(t/=10)|0))*t}function Jr(e,n){if(e&&e.defaultProps)for(var t in n=r({},n),e=e.defaultProps)void 0===n[t]&&(n[t]=e[t]);return n}var Zr={current:null},eo=null,no=null,to=null;function io(){to=no=eo=null}function ro(e){var n=Zr.current;dr(Zr),e.type._context._currentValue=n}function oo(e,n){for(;null!==e;){var t=e.alternate;if(e.childExpirationTime<n)e.childExpirationTime=n,null!==t&&t.childExpirationTime<n&&(t.childExpirationTime=n);else{if(!(null!==t&&t.childExpirationTime<n))break;t.childExpirationTime=n}e=e.return}}function ao(e,n){eo=e,to=no=null,null!==(e=e.dependencies)&&null!==e.firstContext&&(e.expirationTime>=n&&(La=!0),e.firstContext=null)}function lo(e,n){if(to!==e&&!1!==n&&0!==n)if("number"==typeof n&&1073741823!==n||(to=e,n=1073741823),n={context:e,observedBits:n,next:null},null===no){if(null===eo)throw Error(a(308));no=n,eo.dependencies={expirationTime:0,firstContext:n,responders:null}}else no=no.next=n;return e._currentValue}var so=!1;function co(e){e.updateQueue={baseState:e.memoizedState,baseQueue:null,shared:{pending:null},effects:null}}function uo(e,n){e=e.updateQueue,n.updateQueue===e&&(n.updateQueue={baseState:e.baseState,baseQueue:e.baseQueue,shared:e.shared,effects:e.effects})}function po(e,n){return(e={expirationTime:e,suspenseConfig:n,tag:0,payload:null,callback:null,next:null}).next=e}function fo(e,n){if(null!==(e=e.updateQueue)){var t=(e=e.shared).pending;null===t?n.next=n:(n.next=t.next,t.next=n),e.pending=n}}function mo(e,n){var t=e.alternate;null!==t&&uo(t,e),null===(t=(e=e.updateQueue).baseQueue)?(e.baseQueue=n.next=n,n.next=n):(n.next=t.next,t.next=n)}function go(e,n,t,i){var o=e.updateQueue;so=!1;var a=o.baseQueue,l=o.shared.pending;if(null!==l){if(null!==a){var s=a.next;a.next=l.next,l.next=s}a=l,o.shared.pending=null,null!==(s=e.alternate)&&(null!==(s=s.updateQueue)&&(s.baseQueue=l))}if(null!==a){s=a.next;var c=o.baseState,u=0,p=null,d=null,f=null;if(null!==s)for(var m=s;;){if((l=m.expirationTime)<i){var g={expirationTime:m.expirationTime,suspenseConfig:m.suspenseConfig,tag:m.tag,payload:m.payload,callback:m.callback,next:null};null===f?(d=f=g,p=c):f=f.next=g,l>u&&(u=l)}else{null!==f&&(f=f.next={expirationTime:1073741823,suspenseConfig:m.suspenseConfig,tag:m.tag,payload:m.payload,callback:m.callback,next:null}),fs(l,m.suspenseConfig);e:{var h=e,v=m;switch(l=n,g=t,v.tag){case 1:if("function"==typeof(h=v.payload)){c=h.call(g,c,l);break e}c=h;break e;case 3:h.effectTag=-4097&h.effectTag|64;case 0:if(null==(l="function"==typeof(h=v.payload)?h.call(g,c,l):h))break e;c=r({},c,l);break e;case 2:so=!0}}null!==m.callback&&(e.effectTag|=32,null===(l=o.effects)?o.effects=[m]:l.push(m))}if(null===(m=m.next)||m===s){if(null===(l=o.shared.pending))break;m=a.next=l.next,l.next=s,o.baseQueue=a=l,o.shared.pending=null}}null===f?p=c:f.next=d,o.baseState=p,o.baseQueue=f,ms(u),e.expirationTime=u,e.memoizedState=c}}function ho(e,n,t){if(e=n.effects,n.effects=null,null!==e)for(n=0;n<e.length;n++){var i=e[n],r=i.callback;if(null!==r){if(i.callback=null,i=r,r=t,"function"!=typeof i)throw Error(a(191,i));i.call(r)}}}var vo=Q.ReactCurrentBatchConfig,bo=(new i.Component).refs;function yo(e,n,t,i){t=null==(t=t(i,n=e.memoizedState))?n:r({},n,t),e.memoizedState=t,0===e.expirationTime&&(e.updateQueue.baseState=t)}var ko={isMounted:function(e){return!!(e=e._reactInternalFiber)&&en(e)===e},enqueueSetState:function(e,n,t){e=e._reactInternalFiber;var i=es(),r=vo.suspense;(r=po(i=ns(i,e,r),r)).payload=n,null!=t&&(r.callback=t),fo(e,r),ts(e,i)},enqueueReplaceState:function(e,n,t){e=e._reactInternalFiber;var i=es(),r=vo.suspense;(r=po(i=ns(i,e,r),r)).tag=1,r.payload=n,null!=t&&(r.callback=t),fo(e,r),ts(e,i)},enqueueForceUpdate:function(e,n){e=e._reactInternalFiber;var t=es(),i=vo.suspense;(i=po(t=ns(t,e,i),i)).tag=2,null!=n&&(i.callback=n),fo(e,i),ts(e,t)}};function wo(e,n,t,i,r,o,a){return"function"==typeof(e=e.stateNode).shouldComponentUpdate?e.shouldComponentUpdate(i,o,a):!n.prototype||!n.prototype.isPureReactComponent||(!Gi(t,i)||!Gi(r,o))}function _o(e,n,t){var i=!1,r=mr,o=n.contextType;return"object"==typeof o&&null!==o?o=lo(o):(r=yr(n)?vr:gr.current,o=(i=null!=(i=n.contextTypes))?br(e,r):mr),n=new n(t,o),e.memoizedState=null!==n.state&&void 0!==n.state?n.state:null,n.updater=ko,e.stateNode=n,n._reactInternalFiber=e,i&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=r,e.__reactInternalMemoizedMaskedChildContext=o),n}function xo(e,n,t,i){e=n.state,"function"==typeof n.componentWillReceiveProps&&n.componentWillReceiveProps(t,i),"function"==typeof n.UNSAFE_componentWillReceiveProps&&n.UNSAFE_componentWillReceiveProps(t,i),n.state!==e&&ko.enqueueReplaceState(n,n.state,null)}function Eo(e,n,t,i){var r=e.stateNode;r.props=t,r.state=e.memoizedState,r.refs=bo,co(e);var o=n.contextType;"object"==typeof o&&null!==o?r.context=lo(o):(o=yr(n)?vr:gr.current,r.context=br(e,o)),go(e,t,r,i),r.state=e.memoizedState,"function"==typeof(o=n.getDerivedStateFromProps)&&(yo(e,n,o,t),r.state=e.memoizedState),"function"==typeof n.getDerivedStateFromProps||"function"==typeof r.getSnapshotBeforeUpdate||"function"!=typeof r.UNSAFE_componentWillMount&&"function"!=typeof r.componentWillMount||(n=r.state,"function"==typeof r.componentWillMount&&r.componentWillMount(),"function"==typeof r.UNSAFE_componentWillMount&&r.UNSAFE_componentWillMount(),n!==r.state&&ko.enqueueReplaceState(r,r.state,null),go(e,t,r,i),r.state=e.memoizedState),"function"==typeof r.componentDidMount&&(e.effectTag|=4)}var So=Array.isArray;function To(e,n,t){if(null!==(e=t.ref)&&"function"!=typeof e&&"object"!=typeof e){if(t._owner){if(t=t._owner){if(1!==t.tag)throw Error(a(309));var i=t.stateNode}if(!i)throw Error(a(147,e));var r=""+e;return null!==n&&null!==n.ref&&"function"==typeof n.ref&&n.ref._stringRef===r?n.ref:((n=function(e){var n=i.refs;n===bo&&(n=i.refs={}),null===e?delete n[r]:n[r]=e})._stringRef=r,n)}if("string"!=typeof e)throw Error(a(284));if(!t._owner)throw Error(a(290,e))}return e}function Po(e,n){if("textarea"!==e.type)throw Error(a(31,"[object Object]"===Object.prototype.toString.call(n)?"object with keys {"+Object.keys(n).join(", ")+"}":n,""))}function Co(e){function n(n,t){if(e){var i=n.lastEffect;null!==i?(i.nextEffect=t,n.lastEffect=t):n.firstEffect=n.lastEffect=t,t.nextEffect=null,t.effectTag=8}}function t(t,i){if(!e)return null;for(;null!==i;)n(t,i),i=i.sibling;return null}function i(e,n){for(e=new Map;null!==n;)null!==n.key?e.set(n.key,n):e.set(n.index,n),n=n.sibling;return e}function r(e,n){return(e=Ls(e,n)).index=0,e.sibling=null,e}function o(n,t,i){return n.index=i,e?null!==(i=n.alternate)?(i=i.index)<t?(n.effectTag=2,t):i:(n.effectTag=2,t):t}function l(n){return e&&null===n.alternate&&(n.effectTag=2),n}function s(e,n,t,i){return null===n||6!==n.tag?((n=Ms(t,e.mode,i)).return=e,n):((n=r(n,t)).return=e,n)}function c(e,n,t,i){return null!==n&&n.elementType===t.type?((i=r(n,t.props)).ref=To(e,n,t),i.return=e,i):((i=Ds(t.type,t.key,t.props,null,e.mode,i)).ref=To(e,n,t),i.return=e,i)}function u(e,n,t,i){return null===n||4!==n.tag||n.stateNode.containerInfo!==t.containerInfo||n.stateNode.implementation!==t.implementation?((n=Fs(t,e.mode,i)).return=e,n):((n=r(n,t.children||[])).return=e,n)}function p(e,n,t,i,o){return null===n||7!==n.tag?((n=js(t,e.mode,i,o)).return=e,n):((n=r(n,t)).return=e,n)}function d(e,n,t){if("string"==typeof n||"number"==typeof n)return(n=Ms(""+n,e.mode,t)).return=e,n;if("object"==typeof n&&null!==n){switch(n.$$typeof){case ee:return(t=Ds(n.type,n.key,n.props,null,e.mode,t)).ref=To(e,null,n),t.return=e,t;case ne:return(n=Fs(n,e.mode,t)).return=e,n}if(So(n)||ge(n))return(n=js(n,e.mode,t,null)).return=e,n;Po(e,n)}return null}function f(e,n,t,i){var r=null!==n?n.key:null;if("string"==typeof t||"number"==typeof t)return null!==r?null:s(e,n,""+t,i);if("object"==typeof t&&null!==t){switch(t.$$typeof){case ee:return t.key===r?t.type===te?p(e,n,t.props.children,i,r):c(e,n,t,i):null;case ne:return t.key===r?u(e,n,t,i):null}if(So(t)||ge(t))return null!==r?null:p(e,n,t,i,null);Po(e,t)}return null}function m(e,n,t,i,r){if("string"==typeof i||"number"==typeof i)return s(n,e=e.get(t)||null,""+i,r);if("object"==typeof i&&null!==i){switch(i.$$typeof){case ee:return e=e.get(null===i.key?t:i.key)||null,i.type===te?p(n,e,i.props.children,r,i.key):c(n,e,i,r);case ne:return u(n,e=e.get(null===i.key?t:i.key)||null,i,r)}if(So(i)||ge(i))return p(n,e=e.get(t)||null,i,r,null);Po(n,i)}return null}function g(r,a,l,s){for(var c=null,u=null,p=a,g=a=0,h=null;null!==p&&g<l.length;g++){p.index>g?(h=p,p=null):h=p.sibling;var v=f(r,p,l[g],s);if(null===v){null===p&&(p=h);break}e&&p&&null===v.alternate&&n(r,p),a=o(v,a,g),null===u?c=v:u.sibling=v,u=v,p=h}if(g===l.length)return t(r,p),c;if(null===p){for(;g<l.length;g++)null!==(p=d(r,l[g],s))&&(a=o(p,a,g),null===u?c=p:u.sibling=p,u=p);return c}for(p=i(r,p);g<l.length;g++)null!==(h=m(p,r,g,l[g],s))&&(e&&null!==h.alternate&&p.delete(null===h.key?g:h.key),a=o(h,a,g),null===u?c=h:u.sibling=h,u=h);return e&&p.forEach((function(e){return n(r,e)})),c}function h(r,l,s,c){var u=ge(s);if("function"!=typeof u)throw Error(a(150));if(null==(s=u.call(s)))throw Error(a(151));for(var p=u=null,g=l,h=l=0,v=null,b=s.next();null!==g&&!b.done;h++,b=s.next()){g.index>h?(v=g,g=null):v=g.sibling;var y=f(r,g,b.value,c);if(null===y){null===g&&(g=v);break}e&&g&&null===y.alternate&&n(r,g),l=o(y,l,h),null===p?u=y:p.sibling=y,p=y,g=v}if(b.done)return t(r,g),u;if(null===g){for(;!b.done;h++,b=s.next())null!==(b=d(r,b.value,c))&&(l=o(b,l,h),null===p?u=b:p.sibling=b,p=b);return u}for(g=i(r,g);!b.done;h++,b=s.next())null!==(b=m(g,r,h,b.value,c))&&(e&&null!==b.alternate&&g.delete(null===b.key?h:b.key),l=o(b,l,h),null===p?u=b:p.sibling=b,p=b);return e&&g.forEach((function(e){return n(r,e)})),u}return function(e,i,o,s){var c="object"==typeof o&&null!==o&&o.type===te&&null===o.key;c&&(o=o.props.children);var u="object"==typeof o&&null!==o;if(u)switch(o.$$typeof){case ee:e:{for(u=o.key,c=i;null!==c;){if(c.key===u){switch(c.tag){case 7:if(o.type===te){t(e,c.sibling),(i=r(c,o.props.children)).return=e,e=i;break e}break;default:if(c.elementType===o.type){t(e,c.sibling),(i=r(c,o.props)).ref=To(e,c,o),i.return=e,e=i;break e}}t(e,c);break}n(e,c),c=c.sibling}o.type===te?((i=js(o.props.children,e.mode,s,o.key)).return=e,e=i):((s=Ds(o.type,o.key,o.props,null,e.mode,s)).ref=To(e,i,o),s.return=e,e=s)}return l(e);case ne:e:{for(c=o.key;null!==i;){if(i.key===c){if(4===i.tag&&i.stateNode.containerInfo===o.containerInfo&&i.stateNode.implementation===o.implementation){t(e,i.sibling),(i=r(i,o.children||[])).return=e,e=i;break e}t(e,i);break}n(e,i),i=i.sibling}(i=Fs(o,e.mode,s)).return=e,e=i}return l(e)}if("string"==typeof o||"number"==typeof o)return o=""+o,null!==i&&6===i.tag?(t(e,i.sibling),(i=r(i,o)).return=e,e=i):(t(e,i),(i=Ms(o,e.mode,s)).return=e,e=i),l(e);if(So(o))return g(e,i,o,s);if(ge(o))return h(e,i,o,s);if(u&&Po(e,o),void 0===o&&!c)switch(e.tag){case 1:case 0:throw e=e.type,Error(a(152,e.displayName||e.name||"Component"))}return t(e,i)}}var Oo=Co(!0),Ao=Co(!1),Ro={},No={current:Ro},Io={current:Ro},Lo={current:Ro};function Do(e){if(e===Ro)throw Error(a(174));return e}function jo(e,n){switch(fr(Lo,n),fr(Io,e),fr(No,Ro),e=n.nodeType){case 9:case 11:n=(n=n.documentElement)?n.namespaceURI:Me(null,"");break;default:n=Me(n=(e=8===e?n.parentNode:n).namespaceURI||null,e=e.tagName)}dr(No),fr(No,n)}function Mo(){dr(No),dr(Io),dr(Lo)}function Fo(e){Do(Lo.current);var n=Do(No.current),t=Me(n,e.type);n!==t&&(fr(Io,e),fr(No,t))}function Bo(e){Io.current===e&&(dr(No),dr(Io))}var zo={current:0};function Uo(e){for(var n=e;null!==n;){if(13===n.tag){var t=n.memoizedState;if(null!==t&&(null===(t=t.dehydrated)||t.data===vt||t.data===bt))return n}else if(19===n.tag&&void 0!==n.memoizedProps.revealOrder){if(0!=(64&n.effectTag))return n}else if(null!==n.child){n.child.return=n,n=n.child;continue}if(n===e)break;for(;null===n.sibling;){if(null===n.return||n.return===e)return null;n=n.return}n.sibling.return=n.return,n=n.sibling}return null}function $o(e,n){return{responder:e,props:n}}var Go=Q.ReactCurrentDispatcher,qo=Q.ReactCurrentBatchConfig,Ho=0,Wo=null,Vo=null,Ko=null,Yo=!1;function Qo(){throw Error(a(321))}function Xo(e,n){if(null===n)return!1;for(var t=0;t<n.length&&t<e.length;t++)if(!Ui(e[t],n[t]))return!1;return!0}function Jo(e,n,t,i,r,o){if(Ho=o,Wo=n,n.memoizedState=null,n.updateQueue=null,n.expirationTime=0,Go.current=null===e||null===e.memoizedState?wa:_a,e=t(i,r),n.expirationTime===Ho){o=0;do{if(n.expirationTime=0,!(25>o))throw Error(a(301));o+=1,Ko=Vo=null,n.updateQueue=null,Go.current=xa,e=t(i,r)}while(n.expirationTime===Ho)}if(Go.current=ka,n=null!==Vo&&null!==Vo.next,Ho=0,Ko=Vo=Wo=null,Yo=!1,n)throw Error(a(300));return e}function Zo(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return null===Ko?Wo.memoizedState=Ko=e:Ko=Ko.next=e,Ko}function ea(){if(null===Vo){var e=Wo.alternate;e=null!==e?e.memoizedState:null}else e=Vo.next;var n=null===Ko?Wo.memoizedState:Ko.next;if(null!==n)Ko=n,Vo=e;else{if(null===e)throw Error(a(310));e={memoizedState:(Vo=e).memoizedState,baseState:Vo.baseState,baseQueue:Vo.baseQueue,queue:Vo.queue,next:null},null===Ko?Wo.memoizedState=Ko=e:Ko=Ko.next=e}return Ko}function na(e,n){return"function"==typeof n?n(e):n}function ta(e){var n=ea(),t=n.queue;if(null===t)throw Error(a(311));t.lastRenderedReducer=e;var i=Vo,r=i.baseQueue,o=t.pending;if(null!==o){if(null!==r){var l=r.next;r.next=o.next,o.next=l}i.baseQueue=r=o,t.pending=null}if(null!==r){r=r.next,i=i.baseState;var s=l=o=null,c=r;do{var u=c.expirationTime;if(u<Ho){var p={expirationTime:c.expirationTime,suspenseConfig:c.suspenseConfig,action:c.action,eagerReducer:c.eagerReducer,eagerState:c.eagerState,next:null};null===s?(l=s=p,o=i):s=s.next=p,u>Wo.expirationTime&&(Wo.expirationTime=u,ms(u))}else null!==s&&(s=s.next={expirationTime:1073741823,suspenseConfig:c.suspenseConfig,action:c.action,eagerReducer:c.eagerReducer,eagerState:c.eagerState,next:null}),fs(u,c.suspenseConfig),i=c.eagerReducer===e?c.eagerState:e(i,c.action);c=c.next}while(null!==c&&c!==r);null===s?o=i:s.next=l,Ui(i,n.memoizedState)||(La=!0),n.memoizedState=i,n.baseState=o,n.baseQueue=s,t.lastRenderedState=i}return[n.memoizedState,t.dispatch]}function ia(e){var n=ea(),t=n.queue;if(null===t)throw Error(a(311));t.lastRenderedReducer=e;var i=t.dispatch,r=t.pending,o=n.memoizedState;if(null!==r){t.pending=null;var l=r=r.next;do{o=e(o,l.action),l=l.next}while(l!==r);Ui(o,n.memoizedState)||(La=!0),n.memoizedState=o,null===n.baseQueue&&(n.baseState=o),t.lastRenderedState=o}return[o,i]}function ra(e){var n=Zo();return"function"==typeof e&&(e=e()),n.memoizedState=n.baseState=e,e=(e=n.queue={pending:null,dispatch:null,lastRenderedReducer:na,lastRenderedState:e}).dispatch=ya.bind(null,Wo,e),[n.memoizedState,e]}function oa(e,n,t,i){return e={tag:e,create:n,destroy:t,deps:i,next:null},null===(n=Wo.updateQueue)?(n={lastEffect:null},Wo.updateQueue=n,n.lastEffect=e.next=e):null===(t=n.lastEffect)?n.lastEffect=e.next=e:(i=t.next,t.next=e,e.next=i,n.lastEffect=e),e}function aa(){return ea().memoizedState}function la(e,n,t,i){var r=Zo();Wo.effectTag|=e,r.memoizedState=oa(1|n,t,void 0,void 0===i?null:i)}function sa(e,n,t,i){var r=ea();i=void 0===i?null:i;var o=void 0;if(null!==Vo){var a=Vo.memoizedState;if(o=a.destroy,null!==i&&Xo(i,a.deps))return void oa(n,t,o,i)}Wo.effectTag|=e,r.memoizedState=oa(1|n,t,o,i)}function ca(e,n){return la(516,4,e,n)}function ua(e,n){return sa(516,4,e,n)}function pa(e,n){return sa(4,2,e,n)}function da(e,n){return"function"==typeof n?(e=e(),n(e),function(){n(null)}):null!=n?(e=e(),n.current=e,function(){n.current=null}):void 0}function fa(e,n,t){return t=null!=t?t.concat([e]):null,sa(4,2,da.bind(null,n,e),t)}function ma(){}function ga(e,n){return Zo().memoizedState=[e,void 0===n?null:n],e}function ha(e,n){var t=ea();n=void 0===n?null:n;var i=t.memoizedState;return null!==i&&null!==n&&Xo(n,i[1])?i[0]:(t.memoizedState=[e,n],e)}function va(e,n){var t=ea();n=void 0===n?null:n;var i=t.memoizedState;return null!==i&&null!==n&&Xo(n,i[1])?i[0]:(e=e(),t.memoizedState=[e,n],e)}function ba(e,n,t){var i=qr();Wr(98>i?98:i,(function(){e(!0)})),Wr(97<i?97:i,(function(){var i=qo.suspense;qo.suspense=void 0===n?null:n;try{e(!1),t()}finally{qo.suspense=i}}))}function ya(e,n,t){var i=es(),r=vo.suspense;r={expirationTime:i=ns(i,e,r),suspenseConfig:r,action:t,eagerReducer:null,eagerState:null,next:null};var o=n.pending;if(null===o?r.next=r:(r.next=o.next,o.next=r),n.pending=r,o=e.alternate,e===Wo||null!==o&&o===Wo)Yo=!0,r.expirationTime=Ho,Wo.expirationTime=Ho;else{if(0===e.expirationTime&&(null===o||0===o.expirationTime)&&null!==(o=n.lastRenderedReducer))try{var a=n.lastRenderedState,l=o(a,t);if(r.eagerReducer=o,r.eagerState=l,Ui(l,a))return}catch(s){}ts(e,i)}}var ka={readContext:lo,useCallback:Qo,useContext:Qo,useEffect:Qo,useImperativeHandle:Qo,useLayoutEffect:Qo,useMemo:Qo,useReducer:Qo,useRef:Qo,useState:Qo,useDebugValue:Qo,useResponder:Qo,useDeferredValue:Qo,useTransition:Qo},wa={readContext:lo,useCallback:ga,useContext:lo,useEffect:ca,useImperativeHandle:function(e,n,t){return t=null!=t?t.concat([e]):null,la(4,2,da.bind(null,n,e),t)},useLayoutEffect:function(e,n){return la(4,2,e,n)},useMemo:function(e,n){var t=Zo();return n=void 0===n?null:n,e=e(),t.memoizedState=[e,n],e},useReducer:function(e,n,t){var i=Zo();return n=void 0!==t?t(n):n,i.memoizedState=i.baseState=n,e=(e=i.queue={pending:null,dispatch:null,lastRenderedReducer:e,lastRenderedState:n}).dispatch=ya.bind(null,Wo,e),[i.memoizedState,e]},useRef:function(e){return e={current:e},Zo().memoizedState=e},useState:ra,useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ra(e),i=t[0],r=t[1];return ca((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ra(!1),t=n[0];return n=n[1],[ga(ba.bind(null,n,e),[n,e]),t]}},_a={readContext:lo,useCallback:ha,useContext:lo,useEffect:ua,useImperativeHandle:fa,useLayoutEffect:pa,useMemo:va,useReducer:ta,useRef:aa,useState:function(){return ta(na)},useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ta(na),i=t[0],r=t[1];return ua((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ta(na),t=n[0];return n=n[1],[ha(ba.bind(null,n,e),[n,e]),t]}},xa={readContext:lo,useCallback:ha,useContext:lo,useEffect:ua,useImperativeHandle:fa,useLayoutEffect:pa,useMemo:va,useReducer:ia,useRef:aa,useState:function(){return ia(na)},useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ia(na),i=t[0],r=t[1];return ua((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ia(na),t=n[0];return n=n[1],[ha(ba.bind(null,n,e),[n,e]),t]}},Ea=null,Sa=null,Ta=!1;function Pa(e,n){var t=Ns(5,null,null,0);t.elementType="DELETED",t.type="DELETED",t.stateNode=n,t.return=e,t.effectTag=8,null!==e.lastEffect?(e.lastEffect.nextEffect=t,e.lastEffect=t):e.firstEffect=e.lastEffect=t}function Ca(e,n){switch(e.tag){case 5:var t=e.type;return null!==(n=1!==n.nodeType||t.toLowerCase()!==n.nodeName.toLowerCase()?null:n)&&(e.stateNode=n,!0);case 6:return null!==(n=""===e.pendingProps||3!==n.nodeType?null:n)&&(e.stateNode=n,!0);case 13:default:return!1}}function Oa(e){if(Ta){var n=Sa;if(n){var t=n;if(!Ca(e,n)){if(!(n=St(t.nextSibling))||!Ca(e,n))return e.effectTag=-1025&e.effectTag|2,Ta=!1,void(Ea=e);Pa(Ea,t)}Ea=e,Sa=St(n.firstChild)}else e.effectTag=-1025&e.effectTag|2,Ta=!1,Ea=e}}function Aa(e){for(e=e.return;null!==e&&5!==e.tag&&3!==e.tag&&13!==e.tag;)e=e.return;Ea=e}function Ra(e){if(e!==Ea)return!1;if(!Ta)return Aa(e),Ta=!0,!1;var n=e.type;if(5!==e.tag||"head"!==n&&"body"!==n&&!_t(n,e.memoizedProps))for(n=Sa;n;)Pa(e,n),n=St(n.nextSibling);if(Aa(e),13===e.tag){if(!(e=null!==(e=e.memoizedState)?e.dehydrated:null))throw Error(a(317));e:{for(e=e.nextSibling,n=0;e;){if(8===e.nodeType){var t=e.data;if("/$"===t){if(0===n){Sa=St(e.nextSibling);break e}n--}else"$"!==t&&t!==bt&&t!==vt||n++}e=e.nextSibling}Sa=null}}else Sa=Ea?St(e.stateNode.nextSibling):null;return!0}function Na(){Sa=Ea=null,Ta=!1}var Ia=Q.ReactCurrentOwner,La=!1;function Da(e,n,t,i){n.child=null===e?Ao(n,null,t,i):Oo(n,e.child,t,i)}function ja(e,n,t,i,r){t=t.render;var o=n.ref;return ao(n,r),i=Jo(e,n,t,i,o,r),null===e||La?(n.effectTag|=1,Da(e,n,i,r),n.child):(n.updateQueue=e.updateQueue,n.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),Ja(e,n,r))}function Ma(e,n,t,i,r,o){if(null===e){var a=t.type;return"function"!=typeof a||Is(a)||void 0!==a.defaultProps||null!==t.compare||void 0!==t.defaultProps?((e=Ds(t.type,null,i,null,n.mode,o)).ref=n.ref,e.return=n,n.child=e):(n.tag=15,n.type=a,Fa(e,n,a,i,r,o))}return a=e.child,r<o&&(r=a.memoizedProps,(t=null!==(t=t.compare)?t:Gi)(r,i)&&e.ref===n.ref)?Ja(e,n,o):(n.effectTag|=1,(e=Ls(a,i)).ref=n.ref,e.return=n,n.child=e)}function Fa(e,n,t,i,r,o){return null!==e&&Gi(e.memoizedProps,i)&&e.ref===n.ref&&(La=!1,r<o)?(n.expirationTime=e.expirationTime,Ja(e,n,o)):za(e,n,t,i,o)}function Ba(e,n){var t=n.ref;(null===e&&null!==t||null!==e&&e.ref!==t)&&(n.effectTag|=128)}function za(e,n,t,i,r){var o=yr(t)?vr:gr.current;return o=br(n,o),ao(n,r),t=Jo(e,n,t,i,o,r),null===e||La?(n.effectTag|=1,Da(e,n,t,r),n.child):(n.updateQueue=e.updateQueue,n.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),Ja(e,n,r))}function Ua(e,n,t,i,r){if(yr(t)){var o=!0;xr(n)}else o=!1;if(ao(n,r),null===n.stateNode)null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),_o(n,t,i),Eo(n,t,i,r),i=!0;else if(null===e){var a=n.stateNode,l=n.memoizedProps;a.props=l;var s=a.context,c=t.contextType;"object"==typeof c&&null!==c?c=lo(c):c=br(n,c=yr(t)?vr:gr.current);var u=t.getDerivedStateFromProps,p="function"==typeof u||"function"==typeof a.getSnapshotBeforeUpdate;p||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(l!==i||s!==c)&&xo(n,a,i,c),so=!1;var d=n.memoizedState;a.state=d,go(n,i,a,r),s=n.memoizedState,l!==i||d!==s||hr.current||so?("function"==typeof u&&(yo(n,t,u,i),s=n.memoizedState),(l=so||wo(n,t,l,i,d,s,c))?(p||"function"!=typeof a.UNSAFE_componentWillMount&&"function"!=typeof a.componentWillMount||("function"==typeof a.componentWillMount&&a.componentWillMount(),"function"==typeof a.UNSAFE_componentWillMount&&a.UNSAFE_componentWillMount()),"function"==typeof a.componentDidMount&&(n.effectTag|=4)):("function"==typeof a.componentDidMount&&(n.effectTag|=4),n.memoizedProps=i,n.memoizedState=s),a.props=i,a.state=s,a.context=c,i=l):("function"==typeof a.componentDidMount&&(n.effectTag|=4),i=!1)}else a=n.stateNode,uo(e,n),l=n.memoizedProps,a.props=n.type===n.elementType?l:Jr(n.type,l),s=a.context,"object"==typeof(c=t.contextType)&&null!==c?c=lo(c):c=br(n,c=yr(t)?vr:gr.current),(p="function"==typeof(u=t.getDerivedStateFromProps)||"function"==typeof a.getSnapshotBeforeUpdate)||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(l!==i||s!==c)&&xo(n,a,i,c),so=!1,s=n.memoizedState,a.state=s,go(n,i,a,r),d=n.memoizedState,l!==i||s!==d||hr.current||so?("function"==typeof u&&(yo(n,t,u,i),d=n.memoizedState),(u=so||wo(n,t,l,i,s,d,c))?(p||"function"!=typeof a.UNSAFE_componentWillUpdate&&"function"!=typeof a.componentWillUpdate||("function"==typeof a.componentWillUpdate&&a.componentWillUpdate(i,d,c),"function"==typeof a.UNSAFE_componentWillUpdate&&a.UNSAFE_componentWillUpdate(i,d,c)),"function"==typeof a.componentDidUpdate&&(n.effectTag|=4),"function"==typeof a.getSnapshotBeforeUpdate&&(n.effectTag|=256)):("function"!=typeof a.componentDidUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=256),n.memoizedProps=i,n.memoizedState=d),a.props=i,a.state=d,a.context=c,i=u):("function"!=typeof a.componentDidUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=256),i=!1);return $a(e,n,t,i,o,r)}function $a(e,n,t,i,r,o){Ba(e,n);var a=0!=(64&n.effectTag);if(!i&&!a)return r&&Er(n,t,!1),Ja(e,n,o);i=n.stateNode,Ia.current=n;var l=a&&"function"!=typeof t.getDerivedStateFromError?null:i.render();return n.effectTag|=1,null!==e&&a?(n.child=Oo(n,e.child,null,o),n.child=Oo(n,null,l,o)):Da(e,n,l,o),n.memoizedState=i.state,r&&Er(n,t,!0),n.child}function Ga(e){var n=e.stateNode;n.pendingContext?wr(0,n.pendingContext,n.pendingContext!==n.context):n.context&&wr(0,n.context,!1),jo(e,n.containerInfo)}var qa,Ha,Wa,Va={dehydrated:null,retryTime:0};function Ka(e,n,t){var i,r=n.mode,o=n.pendingProps,a=zo.current,l=!1;if((i=0!=(64&n.effectTag))||(i=0!=(2&a)&&(null===e||null!==e.memoizedState)),i?(l=!0,n.effectTag&=-65):null!==e&&null===e.memoizedState||void 0===o.fallback||!0===o.unstable_avoidThisFallback||(a|=1),fr(zo,1&a),null===e){if(void 0!==o.fallback&&Oa(n),l){if(l=o.fallback,(o=js(null,r,0,null)).return=n,0==(2&n.mode))for(e=null!==n.memoizedState?n.child.child:n.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(t=js(l,r,t,null)).return=n,o.sibling=t,n.memoizedState=Va,n.child=o,t}return r=o.children,n.memoizedState=null,n.child=Ao(n,null,r,t)}if(null!==e.memoizedState){if(r=(e=e.child).sibling,l){if(o=o.fallback,(t=Ls(e,e.pendingProps)).return=n,0==(2&n.mode)&&(l=null!==n.memoizedState?n.child.child:n.child)!==e.child)for(t.child=l;null!==l;)l.return=t,l=l.sibling;return(r=Ls(r,o)).return=n,t.sibling=r,t.childExpirationTime=0,n.memoizedState=Va,n.child=t,r}return t=Oo(n,e.child,o.children,t),n.memoizedState=null,n.child=t}if(e=e.child,l){if(l=o.fallback,(o=js(null,r,0,null)).return=n,o.child=e,null!==e&&(e.return=o),0==(2&n.mode))for(e=null!==n.memoizedState?n.child.child:n.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(t=js(l,r,t,null)).return=n,o.sibling=t,t.effectTag|=2,o.childExpirationTime=0,n.memoizedState=Va,n.child=o,t}return n.memoizedState=null,n.child=Oo(n,e,o.children,t)}function Ya(e,n){e.expirationTime<n&&(e.expirationTime=n);var t=e.alternate;null!==t&&t.expirationTime<n&&(t.expirationTime=n),oo(e.return,n)}function Qa(e,n,t,i,r,o){var a=e.memoizedState;null===a?e.memoizedState={isBackwards:n,rendering:null,renderingStartTime:0,last:i,tail:t,tailExpiration:0,tailMode:r,lastEffect:o}:(a.isBackwards=n,a.rendering=null,a.renderingStartTime=0,a.last=i,a.tail=t,a.tailExpiration=0,a.tailMode=r,a.lastEffect=o)}function Xa(e,n,t){var i=n.pendingProps,r=i.revealOrder,o=i.tail;if(Da(e,n,i.children,t),0!=(2&(i=zo.current)))i=1&i|2,n.effectTag|=64;else{if(null!==e&&0!=(64&e.effectTag))e:for(e=n.child;null!==e;){if(13===e.tag)null!==e.memoizedState&&Ya(e,t);else if(19===e.tag)Ya(e,t);else if(null!==e.child){e.child.return=e,e=e.child;continue}if(e===n)break e;for(;null===e.sibling;){if(null===e.return||e.return===n)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}i&=1}if(fr(zo,i),0==(2&n.mode))n.memoizedState=null;else switch(r){case"forwards":for(t=n.child,r=null;null!==t;)null!==(e=t.alternate)&&null===Uo(e)&&(r=t),t=t.sibling;null===(t=r)?(r=n.child,n.child=null):(r=t.sibling,t.sibling=null),Qa(n,!1,r,t,o,n.lastEffect);break;case"backwards":for(t=null,r=n.child,n.child=null;null!==r;){if(null!==(e=r.alternate)&&null===Uo(e)){n.child=r;break}e=r.sibling,r.sibling=t,t=r,r=e}Qa(n,!0,t,null,o,n.lastEffect);break;case"together":Qa(n,!1,null,null,void 0,n.lastEffect);break;default:n.memoizedState=null}return n.child}function Ja(e,n,t){null!==e&&(n.dependencies=e.dependencies);var i=n.expirationTime;if(0!==i&&ms(i),n.childExpirationTime<t)return null;if(null!==e&&n.child!==e.child)throw Error(a(153));if(null!==n.child){for(t=Ls(e=n.child,e.pendingProps),n.child=t,t.return=n;null!==e.sibling;)e=e.sibling,(t=t.sibling=Ls(e,e.pendingProps)).return=n;t.sibling=null}return n.child}function Za(e,n){switch(e.tailMode){case"hidden":n=e.tail;for(var t=null;null!==n;)null!==n.alternate&&(t=n),n=n.sibling;null===t?e.tail=null:t.sibling=null;break;case"collapsed":t=e.tail;for(var i=null;null!==t;)null!==t.alternate&&(i=t),t=t.sibling;null===i?n||null===e.tail?e.tail=null:e.tail.sibling=null:i.sibling=null}}function el(e,n,t){var i=n.pendingProps;switch(n.tag){case 2:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return null;case 1:return yr(n.type)&&kr(),null;case 3:return Mo(),dr(hr),dr(gr),(t=n.stateNode).pendingContext&&(t.context=t.pendingContext,t.pendingContext=null),null!==e&&null!==e.child||!Ra(n)||(n.effectTag|=4),null;case 5:Bo(n),t=Do(Lo.current);var o=n.type;if(null!==e&&null!=n.stateNode)Ha(e,n,o,i,t),e.ref!==n.ref&&(n.effectTag|=128);else{if(!i){if(null===n.stateNode)throw Error(a(166));return null}if(e=Do(No.current),Ra(n)){i=n.stateNode,o=n.type;var l=n.memoizedProps;switch(i[Ct]=n,i[Ot]=l,o){case"iframe":case"object":case"embed":Yn("load",i);break;case"video":case"audio":for(e=0;e<Xe.length;e++)Yn(Xe[e],i);break;case"source":Yn("error",i);break;case"img":case"image":case"link":Yn("error",i),Yn("load",i);break;case"form":Yn("reset",i),Yn("submit",i);break;case"details":Yn("toggle",i);break;case"input":xe(i,l),Yn("invalid",i),ct(t,"onChange");break;case"select":i._wrapperState={wasMultiple:!!l.multiple},Yn("invalid",i),ct(t,"onChange");break;case"textarea":Re(i,l),Yn("invalid",i),ct(t,"onChange")}for(var s in at(o,l),e=null,l)if(l.hasOwnProperty(s)){var c=l[s];"children"===s?"string"==typeof c?i.textContent!==c&&(e=["children",c]):"number"==typeof c&&i.textContent!==""+c&&(e=["children",""+c]):E.hasOwnProperty(s)&&null!=c&&ct(t,s)}switch(o){case"input":ke(i),Te(i,l,!0);break;case"textarea":ke(i),Ie(i);break;case"select":case"option":break;default:"function"==typeof l.onClick&&(i.onclick=ut)}t=e,n.updateQueue=t,null!==t&&(n.effectTag|=4)}else{switch(s=9===t.nodeType?t:t.ownerDocument,e===st&&(e=je(o)),e===st?"script"===o?((e=s.createElement("div")).innerHTML="<script><\/script>",e=e.removeChild(e.firstChild)):"string"==typeof i.is?e=s.createElement(o,{is:i.is}):(e=s.createElement(o),"select"===o&&(s=e,i.multiple?s.multiple=!0:i.size&&(s.size=i.size))):e=s.createElementNS(e,o),e[Ct]=n,e[Ot]=i,qa(e,n),n.stateNode=e,s=lt(o,i),o){case"iframe":case"object":case"embed":Yn("load",e),c=i;break;case"video":case"audio":for(c=0;c<Xe.length;c++)Yn(Xe[c],e);c=i;break;case"source":Yn("error",e),c=i;break;case"img":case"image":case"link":Yn("error",e),Yn("load",e),c=i;break;case"form":Yn("reset",e),Yn("submit",e),c=i;break;case"details":Yn("toggle",e),c=i;break;case"input":xe(e,i),c=_e(e,i),Yn("invalid",e),ct(t,"onChange");break;case"option":c=Ce(e,i);break;case"select":e._wrapperState={wasMultiple:!!i.multiple},c=r({},i,{value:void 0}),Yn("invalid",e),ct(t,"onChange");break;case"textarea":Re(e,i),c=Ae(e,i),Yn("invalid",e),ct(t,"onChange");break;default:c=i}at(o,c);var u=c;for(l in u)if(u.hasOwnProperty(l)){var p=u[l];"style"===l?rt(e,p):"dangerouslySetInnerHTML"===l?null!=(p=p?p.__html:void 0)&&ze(e,p):"children"===l?"string"==typeof p?("textarea"!==o||""!==p)&&Ue(e,p):"number"==typeof p&&Ue(e,""+p):"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&"autoFocus"!==l&&(E.hasOwnProperty(l)?null!=p&&ct(t,l):null!=p&&X(e,l,p,s))}switch(o){case"input":ke(e),Te(e,i,!1);break;case"textarea":ke(e),Ie(e);break;case"option":null!=i.value&&e.setAttribute("value",""+be(i.value));break;case"select":e.multiple=!!i.multiple,null!=(t=i.value)?Oe(e,!!i.multiple,t,!1):null!=i.defaultValue&&Oe(e,!!i.multiple,i.defaultValue,!0);break;default:"function"==typeof c.onClick&&(e.onclick=ut)}wt(o,i)&&(n.effectTag|=4)}null!==n.ref&&(n.effectTag|=128)}return null;case 6:if(e&&null!=n.stateNode)Wa(0,n,e.memoizedProps,i);else{if("string"!=typeof i&&null===n.stateNode)throw Error(a(166));t=Do(Lo.current),Do(No.current),Ra(n)?(t=n.stateNode,i=n.memoizedProps,t[Ct]=n,t.nodeValue!==i&&(n.effectTag|=4)):((t=(9===t.nodeType?t:t.ownerDocument).createTextNode(i))[Ct]=n,n.stateNode=t)}return null;case 13:return dr(zo),i=n.memoizedState,0!=(64&n.effectTag)?(n.expirationTime=t,n):(t=null!==i,i=!1,null===e?void 0!==n.memoizedProps.fallback&&Ra(n):(i=null!==(o=e.memoizedState),t||null===o||null!==(o=e.child.sibling)&&(null!==(l=n.firstEffect)?(n.firstEffect=o,o.nextEffect=l):(n.firstEffect=n.lastEffect=o,o.nextEffect=null),o.effectTag=8)),t&&!i&&0!=(2&n.mode)&&(null===e&&!0!==n.memoizedProps.unstable_avoidThisFallback||0!=(1&zo.current)?Dl===Cl&&(Dl=Ol):(Dl!==Cl&&Dl!==Ol||(Dl=Al),0!==zl&&null!==Nl&&(Us(Nl,Ll),$s(Nl,zl)))),(t||i)&&(n.effectTag|=4),null);case 4:return Mo(),null;case 10:return ro(n),null;case 17:return yr(n.type)&&kr(),null;case 19:if(dr(zo),null===(i=n.memoizedState))return null;if(o=0!=(64&n.effectTag),null===(l=i.rendering)){if(o)Za(i,!1);else if(Dl!==Cl||null!==e&&0!=(64&e.effectTag))for(l=n.child;null!==l;){if(null!==(e=Uo(l))){for(n.effectTag|=64,Za(i,!1),null!==(o=e.updateQueue)&&(n.updateQueue=o,n.effectTag|=4),null===i.lastEffect&&(n.firstEffect=null),n.lastEffect=i.lastEffect,i=n.child;null!==i;)l=t,(o=i).effectTag&=2,o.nextEffect=null,o.firstEffect=null,o.lastEffect=null,null===(e=o.alternate)?(o.childExpirationTime=0,o.expirationTime=l,o.child=null,o.memoizedProps=null,o.memoizedState=null,o.updateQueue=null,o.dependencies=null):(o.childExpirationTime=e.childExpirationTime,o.expirationTime=e.expirationTime,o.child=e.child,o.memoizedProps=e.memoizedProps,o.memoizedState=e.memoizedState,o.updateQueue=e.updateQueue,l=e.dependencies,o.dependencies=null===l?null:{expirationTime:l.expirationTime,firstContext:l.firstContext,responders:l.responders}),i=i.sibling;return fr(zo,1&zo.current|2),n.child}l=l.sibling}}else{if(!o)if(null!==(e=Uo(l))){if(n.effectTag|=64,o=!0,null!==(t=e.updateQueue)&&(n.updateQueue=t,n.effectTag|=4),Za(i,!0),null===i.tail&&"hidden"===i.tailMode&&!l.alternate)return null!==(n=n.lastEffect=i.lastEffect)&&(n.nextEffect=null),null}else 2*Gr()-i.renderingStartTime>i.tailExpiration&&1<t&&(n.effectTag|=64,o=!0,Za(i,!1),n.expirationTime=n.childExpirationTime=t-1);i.isBackwards?(l.sibling=n.child,n.child=l):(null!==(t=i.last)?t.sibling=l:n.child=l,i.last=l)}return null!==i.tail?(0===i.tailExpiration&&(i.tailExpiration=Gr()+500),t=i.tail,i.rendering=t,i.tail=t.sibling,i.lastEffect=n.lastEffect,i.renderingStartTime=Gr(),t.sibling=null,n=zo.current,fr(zo,o?1&n|2:1&n),t):null}throw Error(a(156,n.tag))}function nl(e){switch(e.tag){case 1:yr(e.type)&&kr();var n=e.effectTag;return 4096&n?(e.effectTag=-4097&n|64,e):null;case 3:if(Mo(),dr(hr),dr(gr),0!=(64&(n=e.effectTag)))throw Error(a(285));return e.effectTag=-4097&n|64,e;case 5:return Bo(e),null;case 13:return dr(zo),4096&(n=e.effectTag)?(e.effectTag=-4097&n|64,e):null;case 19:return dr(zo),null;case 4:return Mo(),null;case 10:return ro(e),null;default:return null}}function tl(e,n){return{value:e,source:n,stack:ve(n)}}qa=function(e,n){for(var t=n.child;null!==t;){if(5===t.tag||6===t.tag)e.appendChild(t.stateNode);else if(4!==t.tag&&null!==t.child){t.child.return=t,t=t.child;continue}if(t===n)break;for(;null===t.sibling;){if(null===t.return||t.return===n)return;t=t.return}t.sibling.return=t.return,t=t.sibling}},Ha=function(e,n,t,i,o){var a=e.memoizedProps;if(a!==i){var l,s,c=n.stateNode;switch(Do(No.current),e=null,t){case"input":a=_e(c,a),i=_e(c,i),e=[];break;case"option":a=Ce(c,a),i=Ce(c,i),e=[];break;case"select":a=r({},a,{value:void 0}),i=r({},i,{value:void 0}),e=[];break;case"textarea":a=Ae(c,a),i=Ae(c,i),e=[];break;default:"function"!=typeof a.onClick&&"function"==typeof i.onClick&&(c.onclick=ut)}for(l in at(t,i),t=null,a)if(!i.hasOwnProperty(l)&&a.hasOwnProperty(l)&&null!=a[l])if("style"===l)for(s in c=a[l])c.hasOwnProperty(s)&&(t||(t={}),t[s]="");else"dangerouslySetInnerHTML"!==l&&"children"!==l&&"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&"autoFocus"!==l&&(E.hasOwnProperty(l)?e||(e=[]):(e=e||[]).push(l,null));for(l in i){var u=i[l];if(c=null!=a?a[l]:void 0,i.hasOwnProperty(l)&&u!==c&&(null!=u||null!=c))if("style"===l)if(c){for(s in c)!c.hasOwnProperty(s)||u&&u.hasOwnProperty(s)||(t||(t={}),t[s]="");for(s in u)u.hasOwnProperty(s)&&c[s]!==u[s]&&(t||(t={}),t[s]=u[s])}else t||(e||(e=[]),e.push(l,t)),t=u;else"dangerouslySetInnerHTML"===l?(u=u?u.__html:void 0,c=c?c.__html:void 0,null!=u&&c!==u&&(e=e||[]).push(l,u)):"children"===l?c===u||"string"!=typeof u&&"number"!=typeof u||(e=e||[]).push(l,""+u):"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&(E.hasOwnProperty(l)?(null!=u&&ct(o,l),e||c===u||(e=[])):(e=e||[]).push(l,u))}t&&(e=e||[]).push("style",t),o=e,(n.updateQueue=o)&&(n.effectTag|=4)}},Wa=function(e,n,t,i){t!==i&&(n.effectTag|=4)};var il="function"==typeof WeakSet?WeakSet:Set;function rl(e,n){var t=n.source,i=n.stack;null===i&&null!==t&&(i=ve(t)),null!==t&&he(t.type),n=n.value,null!==e&&1===e.tag&&he(e.type);try{console.error(n)}catch(r){setTimeout((function(){throw r}))}}function ol(e){var n=e.ref;if(null!==n)if("function"==typeof n)try{n(null)}catch(t){Ts(e,t)}else n.current=null}function al(e,n){switch(n.tag){case 0:case 11:case 15:case 22:return;case 1:if(256&n.effectTag&&null!==e){var t=e.memoizedProps,i=e.memoizedState;n=(e=n.stateNode).getSnapshotBeforeUpdate(n.elementType===n.type?t:Jr(n.type,t),i),e.__reactInternalSnapshotBeforeUpdate=n}return;case 3:case 5:case 6:case 4:case 17:return}throw Error(a(163))}function ll(e,n){if(null!==(n=null!==(n=n.updateQueue)?n.lastEffect:null)){var t=n=n.next;do{if((t.tag&e)===e){var i=t.destroy;t.destroy=void 0,void 0!==i&&i()}t=t.next}while(t!==n)}}function sl(e,n){if(null!==(n=null!==(n=n.updateQueue)?n.lastEffect:null)){var t=n=n.next;do{if((t.tag&e)===e){var i=t.create;t.destroy=i()}t=t.next}while(t!==n)}}function cl(e,n,t){switch(t.tag){case 0:case 11:case 15:case 22:return void sl(3,t);case 1:if(e=t.stateNode,4&t.effectTag)if(null===n)e.componentDidMount();else{var i=t.elementType===t.type?n.memoizedProps:Jr(t.type,n.memoizedProps);e.componentDidUpdate(i,n.memoizedState,e.__reactInternalSnapshotBeforeUpdate)}return void(null!==(n=t.updateQueue)&&ho(t,n,e));case 3:if(null!==(n=t.updateQueue)){if(e=null,null!==t.child)switch(t.child.tag){case 5:e=t.child.stateNode;break;case 1:e=t.child.stateNode}ho(t,n,e)}return;case 5:return e=t.stateNode,void(null===n&&4&t.effectTag&&wt(t.type,t.memoizedProps)&&e.focus());case 6:case 4:case 12:return;case 13:return void(null===t.memoizedState&&(t=t.alternate,null!==t&&(t=t.memoizedState,null!==t&&(t=t.dehydrated,null!==t&&Fn(t)))));case 19:case 17:case 20:case 21:return}throw Error(a(163))}function ul(e,n,t){switch("function"==typeof As&&As(n),n.tag){case 0:case 11:case 14:case 15:case 22:if(null!==(e=n.updateQueue)&&null!==(e=e.lastEffect)){var i=e.next;Wr(97<t?97:t,(function(){var e=i;do{var t=e.destroy;if(void 0!==t){var r=n;try{t()}catch(o){Ts(r,o)}}e=e.next}while(e!==i)}))}break;case 1:ol(n),"function"==typeof(t=n.stateNode).componentWillUnmount&&function(e,n){try{n.props=e.memoizedProps,n.state=e.memoizedState,n.componentWillUnmount()}catch(t){Ts(e,t)}}(n,t);break;case 5:ol(n);break;case 4:hl(e,n,t)}}function pl(e){var n=e.alternate;e.return=null,e.child=null,e.memoizedState=null,e.updateQueue=null,e.dependencies=null,e.alternate=null,e.firstEffect=null,e.lastEffect=null,e.pendingProps=null,e.memoizedProps=null,e.stateNode=null,null!==n&&pl(n)}function dl(e){return 5===e.tag||3===e.tag||4===e.tag}function fl(e){e:{for(var n=e.return;null!==n;){if(dl(n)){var t=n;break e}n=n.return}throw Error(a(160))}switch(n=t.stateNode,t.tag){case 5:var i=!1;break;case 3:case 4:n=n.containerInfo,i=!0;break;default:throw Error(a(161))}16&t.effectTag&&(Ue(n,""),t.effectTag&=-17);e:n:for(t=e;;){for(;null===t.sibling;){if(null===t.return||dl(t.return)){t=null;break e}t=t.return}for(t.sibling.return=t.return,t=t.sibling;5!==t.tag&&6!==t.tag&&18!==t.tag;){if(2&t.effectTag)continue n;if(null===t.child||4===t.tag)continue n;t.child.return=t,t=t.child}if(!(2&t.effectTag)){t=t.stateNode;break e}}i?ml(e,t,n):gl(e,t,n)}function ml(e,n,t){var i=e.tag,r=5===i||6===i;if(r)e=r?e.stateNode:e.stateNode.instance,n?8===t.nodeType?t.parentNode.insertBefore(e,n):t.insertBefore(e,n):(8===t.nodeType?(n=t.parentNode).insertBefore(e,t):(n=t).appendChild(e),null!=(t=t._reactRootContainer)||null!==n.onclick||(n.onclick=ut));else if(4!==i&&null!==(e=e.child))for(ml(e,n,t),e=e.sibling;null!==e;)ml(e,n,t),e=e.sibling}function gl(e,n,t){var i=e.tag,r=5===i||6===i;if(r)e=r?e.stateNode:e.stateNode.instance,n?t.insertBefore(e,n):t.appendChild(e);else if(4!==i&&null!==(e=e.child))for(gl(e,n,t),e=e.sibling;null!==e;)gl(e,n,t),e=e.sibling}function hl(e,n,t){for(var i,r,o=n,l=!1;;){if(!l){l=o.return;e:for(;;){if(null===l)throw Error(a(160));switch(i=l.stateNode,l.tag){case 5:r=!1;break e;case 3:case 4:i=i.containerInfo,r=!0;break e}l=l.return}l=!0}if(5===o.tag||6===o.tag){e:for(var s=e,c=o,u=t,p=c;;)if(ul(s,p,u),null!==p.child&&4!==p.tag)p.child.return=p,p=p.child;else{if(p===c)break e;for(;null===p.sibling;){if(null===p.return||p.return===c)break e;p=p.return}p.sibling.return=p.return,p=p.sibling}r?(s=i,c=o.stateNode,8===s.nodeType?s.parentNode.removeChild(c):s.removeChild(c)):i.removeChild(o.stateNode)}else if(4===o.tag){if(null!==o.child){i=o.stateNode.containerInfo,r=!0,o.child.return=o,o=o.child;continue}}else if(ul(e,o,t),null!==o.child){o.child.return=o,o=o.child;continue}if(o===n)break;for(;null===o.sibling;){if(null===o.return||o.return===n)return;4===(o=o.return).tag&&(l=!1)}o.sibling.return=o.return,o=o.sibling}}function vl(e,n){switch(n.tag){case 0:case 11:case 14:case 15:case 22:return void ll(3,n);case 1:return;case 5:var t=n.stateNode;if(null!=t){var i=n.memoizedProps,r=null!==e?e.memoizedProps:i;e=n.type;var o=n.updateQueue;if(n.updateQueue=null,null!==o){for(t[Ot]=i,"input"===e&&"radio"===i.type&&null!=i.name&&Ee(t,i),lt(e,r),n=lt(e,i),r=0;r<o.length;r+=2){var l=o[r],s=o[r+1];"style"===l?rt(t,s):"dangerouslySetInnerHTML"===l?ze(t,s):"children"===l?Ue(t,s):X(t,l,s,n)}switch(e){case"input":Se(t,i);break;case"textarea":Ne(t,i);break;case"select":n=t._wrapperState.wasMultiple,t._wrapperState.wasMultiple=!!i.multiple,null!=(e=i.value)?Oe(t,!!i.multiple,e,!1):n!==!!i.multiple&&(null!=i.defaultValue?Oe(t,!!i.multiple,i.defaultValue,!0):Oe(t,!!i.multiple,i.multiple?[]:"",!1))}}}return;case 6:if(null===n.stateNode)throw Error(a(162));return void(n.stateNode.nodeValue=n.memoizedProps);case 3:return void((n=n.stateNode).hydrate&&(n.hydrate=!1,Fn(n.containerInfo)));case 12:return;case 13:if(t=n,null===n.memoizedState?i=!1:(i=!0,t=n.child,$l=Gr()),null!==t)e:for(e=t;;){if(5===e.tag)o=e.stateNode,i?"function"==typeof(o=o.style).setProperty?o.setProperty("display","none","important"):o.display="none":(o=e.stateNode,r=null!=(r=e.memoizedProps.style)&&r.hasOwnProperty("display")?r.display:null,o.style.display=it("display",r));else if(6===e.tag)e.stateNode.nodeValue=i?"":e.memoizedProps;else{if(13===e.tag&&null!==e.memoizedState&&null===e.memoizedState.dehydrated){(o=e.child.sibling).return=e,e=o;continue}if(null!==e.child){e.child.return=e,e=e.child;continue}}if(e===t)break;for(;null===e.sibling;){if(null===e.return||e.return===t)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}return void bl(n);case 19:return void bl(n);case 17:return}throw Error(a(163))}function bl(e){var n=e.updateQueue;if(null!==n){e.updateQueue=null;var t=e.stateNode;null===t&&(t=e.stateNode=new il),n.forEach((function(n){var i=Cs.bind(null,e,n);t.has(n)||(t.add(n),n.then(i,i))}))}}var yl="function"==typeof WeakMap?WeakMap:Map;function kl(e,n,t){(t=po(t,null)).tag=3,t.payload={element:null};var i=n.value;return t.callback=function(){ql||(ql=!0,Hl=i),rl(e,n)},t}function wl(e,n,t){(t=po(t,null)).tag=3;var i=e.type.getDerivedStateFromError;if("function"==typeof i){var r=n.value;t.payload=function(){return rl(e,n),i(r)}}var o=e.stateNode;return null!==o&&"function"==typeof o.componentDidCatch&&(t.callback=function(){"function"!=typeof i&&(null===Wl?Wl=new Set([this]):Wl.add(this),rl(e,n));var t=n.stack;this.componentDidCatch(n.value,{componentStack:null!==t?t:""})}),t}var _l,xl=Math.ceil,El=Q.ReactCurrentDispatcher,Sl=Q.ReactCurrentOwner,Tl=16,Pl=32,Cl=0,Ol=3,Al=4,Rl=0,Nl=null,Il=null,Ll=0,Dl=Cl,jl=null,Ml=1073741823,Fl=1073741823,Bl=null,zl=0,Ul=!1,$l=0,Gl=null,ql=!1,Hl=null,Wl=null,Vl=!1,Kl=null,Yl=90,Ql=null,Xl=0,Jl=null,Zl=0;function es(){return 0!=(48&Rl)?1073741821-(Gr()/10|0):0!==Zl?Zl:Zl=1073741821-(Gr()/10|0)}function ns(e,n,t){if(0==(2&(n=n.mode)))return 1073741823;var i=qr();if(0==(4&n))return 99===i?1073741823:1073741822;if(0!=(Rl&Tl))return Ll;if(null!==t)e=Xr(e,0|t.timeoutMs||5e3,250);else switch(i){case 99:e=1073741823;break;case 98:e=Xr(e,150,100);break;case 97:case 96:e=Xr(e,5e3,250);break;case 95:e=2;break;default:throw Error(a(326))}return null!==Nl&&e===Ll&&--e,e}function ts(e,n){if(50<Xl)throw Xl=0,Jl=null,Error(a(185));if(null!==(e=is(e,n))){var t=qr();1073741823===n?0!=(8&Rl)&&0==(48&Rl)?ls(e):(os(e),0===Rl&&Yr()):os(e),0==(4&Rl)||98!==t&&99!==t||(null===Ql?Ql=new Map([[e,n]]):(void 0===(t=Ql.get(e))||t>n)&&Ql.set(e,n))}}function is(e,n){e.expirationTime<n&&(e.expirationTime=n);var t=e.alternate;null!==t&&t.expirationTime<n&&(t.expirationTime=n);var i=e.return,r=null;if(null===i&&3===e.tag)r=e.stateNode;else for(;null!==i;){if(t=i.alternate,i.childExpirationTime<n&&(i.childExpirationTime=n),null!==t&&t.childExpirationTime<n&&(t.childExpirationTime=n),null===i.return&&3===i.tag){r=i.stateNode;break}i=i.return}return null!==r&&(Nl===r&&(ms(n),Dl===Al&&Us(r,Ll)),$s(r,n)),r}function rs(e){var n=e.lastExpiredTime;if(0!==n)return n;if(!zs(e,n=e.firstPendingTime))return n;var t=e.lastPingedTime;return 2>=(e=t>(e=e.nextKnownPendingLevel)?t:e)&&n!==e?0:e}function os(e){if(0!==e.lastExpiredTime)e.callbackExpirationTime=1073741823,e.callbackPriority=99,e.callbackNode=Kr(ls.bind(null,e));else{var n=rs(e),t=e.callbackNode;if(0===n)null!==t&&(e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90);else{var i=es();if(1073741823===n?i=99:1===n||2===n?i=95:i=0>=(i=10*(1073741821-n)-10*(1073741821-i))?99:250>=i?98:5250>=i?97:95,null!==t){var r=e.callbackPriority;if(e.callbackExpirationTime===n&&r>=i)return;t!==jr&&Pr(t)}e.callbackExpirationTime=n,e.callbackPriority=i,n=1073741823===n?Kr(ls.bind(null,e)):Vr(i,as.bind(null,e),{timeout:10*(1073741821-n)-Gr()}),e.callbackNode=n}}}function as(e,n){if(Zl=0,n)return Gs(e,n=es()),os(e),null;var t=rs(e);if(0!==t){if(n=e.callbackNode,0!=(48&Rl))throw Error(a(327));if(xs(),e===Nl&&t===Ll||us(e,t),null!==Il){var i=Rl;Rl|=Tl;for(var r=ds();;)try{hs();break}catch(s){ps(e,s)}if(io(),Rl=i,El.current=r,1===Dl)throw n=jl,us(e,t),Us(e,t),os(e),n;if(null===Il)switch(r=e.finishedWork=e.current.alternate,e.finishedExpirationTime=t,i=Dl,Nl=null,i){case Cl:case 1:throw Error(a(345));case 2:Gs(e,2<t?2:t);break;case Ol:if(Us(e,t),t===(i=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=ys(r)),1073741823===Ml&&10<(r=$l+500-Gr())){if(Ul){var o=e.lastPingedTime;if(0===o||o>=t){e.lastPingedTime=t,us(e,t);break}}if(0!==(o=rs(e))&&o!==t)break;if(0!==i&&i!==t){e.lastPingedTime=i;break}e.timeoutHandle=xt(ks.bind(null,e),r);break}ks(e);break;case Al:if(Us(e,t),t===(i=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=ys(r)),Ul&&(0===(r=e.lastPingedTime)||r>=t)){e.lastPingedTime=t,us(e,t);break}if(0!==(r=rs(e))&&r!==t)break;if(0!==i&&i!==t){e.lastPingedTime=i;break}if(1073741823!==Fl?i=10*(1073741821-Fl)-Gr():1073741823===Ml?i=0:(i=10*(1073741821-Ml)-5e3,0>(i=(r=Gr())-i)&&(i=0),(t=10*(1073741821-t)-r)<(i=(120>i?120:480>i?480:1080>i?1080:1920>i?1920:3e3>i?3e3:4320>i?4320:1960*xl(i/1960))-i)&&(i=t)),10<i){e.timeoutHandle=xt(ks.bind(null,e),i);break}ks(e);break;case 5:if(1073741823!==Ml&&null!==Bl){o=Ml;var l=Bl;if(0>=(i=0|l.busyMinDurationMs)?i=0:(r=0|l.busyDelayMs,i=(o=Gr()-(10*(1073741821-o)-(0|l.timeoutMs||5e3)))<=r?0:r+i-o),10<i){Us(e,t),e.timeoutHandle=xt(ks.bind(null,e),i);break}}ks(e);break;default:throw Error(a(329))}if(os(e),e.callbackNode===n)return as.bind(null,e)}}return null}function ls(e){var n=e.lastExpiredTime;if(n=0!==n?n:1073741823,0!=(48&Rl))throw Error(a(327));if(xs(),e===Nl&&n===Ll||us(e,n),null!==Il){var t=Rl;Rl|=Tl;for(var i=ds();;)try{gs();break}catch(r){ps(e,r)}if(io(),Rl=t,El.current=i,1===Dl)throw t=jl,us(e,n),Us(e,n),os(e),t;if(null!==Il)throw Error(a(261));e.finishedWork=e.current.alternate,e.finishedExpirationTime=n,Nl=null,ks(e),os(e)}return null}function ss(e,n){var t=Rl;Rl|=1;try{return e(n)}finally{0===(Rl=t)&&Yr()}}function cs(e,n){var t=Rl;Rl&=-2,Rl|=8;try{return e(n)}finally{0===(Rl=t)&&Yr()}}function us(e,n){e.finishedWork=null,e.finishedExpirationTime=0;var t=e.timeoutHandle;if(-1!==t&&(e.timeoutHandle=-1,Et(t)),null!==Il)for(t=Il.return;null!==t;){var i=t;switch(i.tag){case 1:null!=(i=i.type.childContextTypes)&&kr();break;case 3:Mo(),dr(hr),dr(gr);break;case 5:Bo(i);break;case 4:Mo();break;case 13:case 19:dr(zo);break;case 10:ro(i)}t=t.return}Nl=e,Il=Ls(e.current,null),Ll=n,Dl=Cl,jl=null,Fl=Ml=1073741823,Bl=null,zl=0,Ul=!1}function ps(e,n){for(;;){try{if(io(),Go.current=ka,Yo)for(var t=Wo.memoizedState;null!==t;){var i=t.queue;null!==i&&(i.pending=null),t=t.next}if(Ho=0,Ko=Vo=Wo=null,Yo=!1,null===Il||null===Il.return)return Dl=1,jl=n,Il=null;e:{var r=e,o=Il.return,a=Il,l=n;if(n=Ll,a.effectTag|=2048,a.firstEffect=a.lastEffect=null,null!==l&&"object"==typeof l&&"function"==typeof l.then){var s=l;if(0==(2&a.mode)){var c=a.alternate;c?(a.updateQueue=c.updateQueue,a.memoizedState=c.memoizedState,a.expirationTime=c.expirationTime):(a.updateQueue=null,a.memoizedState=null)}var u=0!=(1&zo.current),p=o;do{var d;if(d=13===p.tag){var f=p.memoizedState;if(null!==f)d=null!==f.dehydrated;else{var m=p.memoizedProps;d=void 0!==m.fallback&&(!0!==m.unstable_avoidThisFallback||!u)}}if(d){var g=p.updateQueue;if(null===g){var h=new Set;h.add(s),p.updateQueue=h}else g.add(s);if(0==(2&p.mode)){if(p.effectTag|=64,a.effectTag&=-2981,1===a.tag)if(null===a.alternate)a.tag=17;else{var v=po(1073741823,null);v.tag=2,fo(a,v)}a.expirationTime=1073741823;break e}l=void 0,a=n;var b=r.pingCache;if(null===b?(b=r.pingCache=new yl,l=new Set,b.set(s,l)):void 0===(l=b.get(s))&&(l=new Set,b.set(s,l)),!l.has(a)){l.add(a);var y=Ps.bind(null,r,s,a);s.then(y,y)}p.effectTag|=4096,p.expirationTime=n;break e}p=p.return}while(null!==p);l=Error((he(a.type)||"A React component")+" suspended while rendering, but no fallback UI was specified.\n\nAdd a <Suspense fallback=...> component higher in the tree to provide a loading indicator or placeholder to display."+ve(a))}5!==Dl&&(Dl=2),l=tl(l,a),p=o;do{switch(p.tag){case 3:s=l,p.effectTag|=4096,p.expirationTime=n,mo(p,kl(p,s,n));break e;case 1:s=l;var k=p.type,w=p.stateNode;if(0==(64&p.effectTag)&&("function"==typeof k.getDerivedStateFromError||null!==w&&"function"==typeof w.componentDidCatch&&(null===Wl||!Wl.has(w)))){p.effectTag|=4096,p.expirationTime=n,mo(p,wl(p,s,n));break e}}p=p.return}while(null!==p)}Il=bs(Il)}catch(_){n=_;continue}break}}function ds(){var e=El.current;return El.current=ka,null===e?ka:e}function fs(e,n){e<Ml&&2<e&&(Ml=e),null!==n&&e<Fl&&2<e&&(Fl=e,Bl=n)}function ms(e){e>zl&&(zl=e)}function gs(){for(;null!==Il;)Il=vs(Il)}function hs(){for(;null!==Il&&!Mr();)Il=vs(Il)}function vs(e){var n=_l(e.alternate,e,Ll);return e.memoizedProps=e.pendingProps,null===n&&(n=bs(e)),Sl.current=null,n}function bs(e){Il=e;do{var n=Il.alternate;if(e=Il.return,0==(2048&Il.effectTag)){if(n=el(n,Il,Ll),1===Ll||1!==Il.childExpirationTime){for(var t=0,i=Il.child;null!==i;){var r=i.expirationTime,o=i.childExpirationTime;r>t&&(t=r),o>t&&(t=o),i=i.sibling}Il.childExpirationTime=t}if(null!==n)return n;null!==e&&0==(2048&e.effectTag)&&(null===e.firstEffect&&(e.firstEffect=Il.firstEffect),null!==Il.lastEffect&&(null!==e.lastEffect&&(e.lastEffect.nextEffect=Il.firstEffect),e.lastEffect=Il.lastEffect),1<Il.effectTag&&(null!==e.lastEffect?e.lastEffect.nextEffect=Il:e.firstEffect=Il,e.lastEffect=Il))}else{if(null!==(n=nl(Il)))return n.effectTag&=2047,n;null!==e&&(e.firstEffect=e.lastEffect=null,e.effectTag|=2048)}if(null!==(n=Il.sibling))return n;Il=e}while(null!==Il);return Dl===Cl&&(Dl=5),null}function ys(e){var n=e.expirationTime;return n>(e=e.childExpirationTime)?n:e}function ks(e){var n=qr();return Wr(99,ws.bind(null,e,n)),null}function ws(e,n){do{xs()}while(null!==Kl);if(0!=(48&Rl))throw Error(a(327));var t=e.finishedWork,i=e.finishedExpirationTime;if(null===t)return null;if(e.finishedWork=null,e.finishedExpirationTime=0,t===e.current)throw Error(a(177));e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90,e.nextKnownPendingLevel=0;var r=ys(t);if(e.firstPendingTime=r,i<=e.lastSuspendedTime?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:i<=e.firstSuspendedTime&&(e.firstSuspendedTime=i-1),i<=e.lastPingedTime&&(e.lastPingedTime=0),i<=e.lastExpiredTime&&(e.lastExpiredTime=0),e===Nl&&(Il=Nl=null,Ll=0),1<t.effectTag?null!==t.lastEffect?(t.lastEffect.nextEffect=t,r=t.firstEffect):r=t:r=t.firstEffect,null!==r){var o=Rl;Rl|=Pl,Sl.current=null,yt=Kn;var l=gt();if(ht(l)){if("selectionStart"in l)var s={start:l.selectionStart,end:l.selectionEnd};else e:{var c=(s=(s=l.ownerDocument)&&s.defaultView||window).getSelection&&s.getSelection();if(c&&0!==c.rangeCount){s=c.anchorNode;var u=c.anchorOffset,p=c.focusNode;c=c.focusOffset;try{s.nodeType,p.nodeType}catch(T){s=null;break e}var d=0,f=-1,m=-1,g=0,h=0,v=l,b=null;n:for(;;){for(var y;v!==s||0!==u&&3!==v.nodeType||(f=d+u),v!==p||0!==c&&3!==v.nodeType||(m=d+c),3===v.nodeType&&(d+=v.nodeValue.length),null!==(y=v.firstChild);)b=v,v=y;for(;;){if(v===l)break n;if(b===s&&++g===u&&(f=d),b===p&&++h===c&&(m=d),null!==(y=v.nextSibling))break;b=(v=b).parentNode}v=y}s=-1===f||-1===m?null:{start:f,end:m}}else s=null}s=s||{start:0,end:0}}else s=null;kt={activeElementDetached:null,focusedElem:l,selectionRange:s},Kn=!1,Gl=r;do{try{_s()}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);Gl=r;do{try{for(l=e,s=n;null!==Gl;){var k=Gl.effectTag;if(16&k&&Ue(Gl.stateNode,""),128&k){var w=Gl.alternate;if(null!==w){var _=w.ref;null!==_&&("function"==typeof _?_(null):_.current=null)}}switch(1038&k){case 2:fl(Gl),Gl.effectTag&=-3;break;case 6:fl(Gl),Gl.effectTag&=-3,vl(Gl.alternate,Gl);break;case 1024:Gl.effectTag&=-1025;break;case 1028:Gl.effectTag&=-1025,vl(Gl.alternate,Gl);break;case 4:vl(Gl.alternate,Gl);break;case 8:hl(l,u=Gl,s),pl(u)}Gl=Gl.nextEffect}}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);if(_=kt,w=gt(),k=_.focusedElem,s=_.selectionRange,w!==k&&k&&k.ownerDocument&&mt(k.ownerDocument.documentElement,k)){null!==s&&ht(k)&&(w=s.start,void 0===(_=s.end)&&(_=w),"selectionStart"in k?(k.selectionStart=w,k.selectionEnd=Math.min(_,k.value.length)):(_=(w=k.ownerDocument||document)&&w.defaultView||window).getSelection&&(_=_.getSelection(),u=k.textContent.length,l=Math.min(s.start,u),s=void 0===s.end?l:Math.min(s.end,u),!_.extend&&l>s&&(u=s,s=l,l=u),u=ft(k,l),p=ft(k,s),u&&p&&(1!==_.rangeCount||_.anchorNode!==u.node||_.anchorOffset!==u.offset||_.focusNode!==p.node||_.focusOffset!==p.offset)&&((w=w.createRange()).setStart(u.node,u.offset),_.removeAllRanges(),l>s?(_.addRange(w),_.extend(p.node,p.offset)):(w.setEnd(p.node,p.offset),_.addRange(w))))),w=[];for(_=k;_=_.parentNode;)1===_.nodeType&&w.push({element:_,left:_.scrollLeft,top:_.scrollTop});for("function"==typeof k.focus&&k.focus(),k=0;k<w.length;k++)(_=w[k]).element.scrollLeft=_.left,_.element.scrollTop=_.top}Kn=!!yt,kt=yt=null,e.current=t,Gl=r;do{try{for(k=e;null!==Gl;){var x=Gl.effectTag;if(36&x&&cl(k,Gl.alternate,Gl),128&x){w=void 0;var E=Gl.ref;if(null!==E){var S=Gl.stateNode;switch(Gl.tag){case 5:w=S;break;default:w=S}"function"==typeof E?E(w):E.current=w}}Gl=Gl.nextEffect}}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);Gl=null,Fr(),Rl=o}else e.current=t;if(Vl)Vl=!1,Kl=e,Yl=n;else for(Gl=r;null!==Gl;)n=Gl.nextEffect,Gl.nextEffect=null,Gl=n;if(0===(n=e.firstPendingTime)&&(Wl=null),1073741823===n?e===Jl?Xl++:(Xl=0,Jl=e):Xl=0,"function"==typeof Os&&Os(t.stateNode,i),os(e),ql)throw ql=!1,e=Hl,Hl=null,e;return 0!=(8&Rl)||Yr(),null}function _s(){for(;null!==Gl;){var e=Gl.effectTag;0!=(256&e)&&al(Gl.alternate,Gl),0==(512&e)||Vl||(Vl=!0,Vr(97,(function(){return xs(),null}))),Gl=Gl.nextEffect}}function xs(){if(90!==Yl){var e=97<Yl?97:Yl;return Yl=90,Wr(e,Es)}}function Es(){if(null===Kl)return!1;var e=Kl;if(Kl=null,0!=(48&Rl))throw Error(a(331));var n=Rl;for(Rl|=Pl,e=e.current.firstEffect;null!==e;){try{var t=e;if(0!=(512&t.effectTag))switch(t.tag){case 0:case 11:case 15:case 22:ll(5,t),sl(5,t)}}catch(i){if(null===e)throw Error(a(330));Ts(e,i)}t=e.nextEffect,e.nextEffect=null,e=t}return Rl=n,Yr(),!0}function Ss(e,n,t){fo(e,n=kl(e,n=tl(t,n),1073741823)),null!==(e=is(e,1073741823))&&os(e)}function Ts(e,n){if(3===e.tag)Ss(e,e,n);else for(var t=e.return;null!==t;){if(3===t.tag){Ss(t,e,n);break}if(1===t.tag){var i=t.stateNode;if("function"==typeof t.type.getDerivedStateFromError||"function"==typeof i.componentDidCatch&&(null===Wl||!Wl.has(i))){fo(t,e=wl(t,e=tl(n,e),1073741823)),null!==(t=is(t,1073741823))&&os(t);break}}t=t.return}}function Ps(e,n,t){var i=e.pingCache;null!==i&&i.delete(n),Nl===e&&Ll===t?Dl===Al||Dl===Ol&&1073741823===Ml&&Gr()-$l<500?us(e,Ll):Ul=!0:zs(e,t)&&(0!==(n=e.lastPingedTime)&&n<t||(e.lastPingedTime=t,os(e)))}function Cs(e,n){var t=e.stateNode;null!==t&&t.delete(n),0===(n=0)&&(n=ns(n=es(),e,null)),null!==(e=is(e,n))&&os(e)}_l=function(e,n,t){var i=n.expirationTime;if(null!==e){var r=n.pendingProps;if(e.memoizedProps!==r||hr.current)La=!0;else{if(i<t){switch(La=!1,n.tag){case 3:Ga(n),Na();break;case 5:if(Fo(n),4&n.mode&&1!==t&&r.hidden)return n.expirationTime=n.childExpirationTime=1,null;break;case 1:yr(n.type)&&xr(n);break;case 4:jo(n,n.stateNode.containerInfo);break;case 10:i=n.memoizedProps.value,r=n.type._context,fr(Zr,r._currentValue),r._currentValue=i;break;case 13:if(null!==n.memoizedState)return 0!==(i=n.child.childExpirationTime)&&i>=t?Ka(e,n,t):(fr(zo,1&zo.current),null!==(n=Ja(e,n,t))?n.sibling:null);fr(zo,1&zo.current);break;case 19:if(i=n.childExpirationTime>=t,0!=(64&e.effectTag)){if(i)return Xa(e,n,t);n.effectTag|=64}if(null!==(r=n.memoizedState)&&(r.rendering=null,r.tail=null),fr(zo,zo.current),!i)return null}return Ja(e,n,t)}La=!1}}else La=!1;switch(n.expirationTime=0,n.tag){case 2:if(i=n.type,null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),e=n.pendingProps,r=br(n,gr.current),ao(n,t),r=Jo(null,n,i,e,r,t),n.effectTag|=1,"object"==typeof r&&null!==r&&"function"==typeof r.render&&void 0===r.$$typeof){if(n.tag=1,n.memoizedState=null,n.updateQueue=null,yr(i)){var o=!0;xr(n)}else o=!1;n.memoizedState=null!==r.state&&void 0!==r.state?r.state:null,co(n);var l=i.getDerivedStateFromProps;"function"==typeof l&&yo(n,i,l,e),r.updater=ko,n.stateNode=r,r._reactInternalFiber=n,Eo(n,i,e,t),n=$a(null,n,i,!0,o,t)}else n.tag=0,Da(null,n,r,t),n=n.child;return n;case 16:e:{if(r=n.elementType,null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),e=n.pendingProps,function(e){if(-1===e._status){e._status=0;var n=e._ctor;n=n(),e._result=n,n.then((function(n){0===e._status&&(n=n.default,e._status=1,e._result=n)}),(function(n){0===e._status&&(e._status=2,e._result=n)}))}}(r),1!==r._status)throw r._result;switch(r=r._result,n.type=r,o=n.tag=function(e){if("function"==typeof e)return Is(e)?1:0;if(null!=e){if((e=e.$$typeof)===se)return 11;if(e===pe)return 14}return 2}(r),e=Jr(r,e),o){case 0:n=za(null,n,r,e,t);break e;case 1:n=Ua(null,n,r,e,t);break e;case 11:n=ja(null,n,r,e,t);break e;case 14:n=Ma(null,n,r,Jr(r.type,e),i,t);break e}throw Error(a(306,r,""))}return n;case 0:return i=n.type,r=n.pendingProps,za(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 1:return i=n.type,r=n.pendingProps,Ua(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 3:if(Ga(n),i=n.updateQueue,null===e||null===i)throw Error(a(282));if(i=n.pendingProps,r=null!==(r=n.memoizedState)?r.element:null,uo(e,n),go(n,i,null,t),(i=n.memoizedState.element)===r)Na(),n=Ja(e,n,t);else{if((r=n.stateNode.hydrate)&&(Sa=St(n.stateNode.containerInfo.firstChild),Ea=n,r=Ta=!0),r)for(t=Ao(n,null,i,t),n.child=t;t;)t.effectTag=-3&t.effectTag|1024,t=t.sibling;else Da(e,n,i,t),Na();n=n.child}return n;case 5:return Fo(n),null===e&&Oa(n),i=n.type,r=n.pendingProps,o=null!==e?e.memoizedProps:null,l=r.children,_t(i,r)?l=null:null!==o&&_t(i,o)&&(n.effectTag|=16),Ba(e,n),4&n.mode&&1!==t&&r.hidden?(n.expirationTime=n.childExpirationTime=1,n=null):(Da(e,n,l,t),n=n.child),n;case 6:return null===e&&Oa(n),null;case 13:return Ka(e,n,t);case 4:return jo(n,n.stateNode.containerInfo),i=n.pendingProps,null===e?n.child=Oo(n,null,i,t):Da(e,n,i,t),n.child;case 11:return i=n.type,r=n.pendingProps,ja(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 7:return Da(e,n,n.pendingProps,t),n.child;case 8:case 12:return Da(e,n,n.pendingProps.children,t),n.child;case 10:e:{i=n.type._context,r=n.pendingProps,l=n.memoizedProps,o=r.value;var s=n.type._context;if(fr(Zr,s._currentValue),s._currentValue=o,null!==l)if(s=l.value,0===(o=Ui(s,o)?0:0|("function"==typeof i._calculateChangedBits?i._calculateChangedBits(s,o):1073741823))){if(l.children===r.children&&!hr.current){n=Ja(e,n,t);break e}}else for(null!==(s=n.child)&&(s.return=n);null!==s;){var c=s.dependencies;if(null!==c){l=s.child;for(var u=c.firstContext;null!==u;){if(u.context===i&&0!=(u.observedBits&o)){1===s.tag&&((u=po(t,null)).tag=2,fo(s,u)),s.expirationTime<t&&(s.expirationTime=t),null!==(u=s.alternate)&&u.expirationTime<t&&(u.expirationTime=t),oo(s.return,t),c.expirationTime<t&&(c.expirationTime=t);break}u=u.next}}else l=10===s.tag&&s.type===n.type?null:s.child;if(null!==l)l.return=s;else for(l=s;null!==l;){if(l===n){l=null;break}if(null!==(s=l.sibling)){s.return=l.return,l=s;break}l=l.return}s=l}Da(e,n,r.children,t),n=n.child}return n;case 9:return r=n.type,i=(o=n.pendingProps).children,ao(n,t),i=i(r=lo(r,o.unstable_observedBits)),n.effectTag|=1,Da(e,n,i,t),n.child;case 14:return o=Jr(r=n.type,n.pendingProps),Ma(e,n,r,o=Jr(r.type,o),i,t);case 15:return Fa(e,n,n.type,n.pendingProps,i,t);case 17:return i=n.type,r=n.pendingProps,r=n.elementType===i?r:Jr(i,r),null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),n.tag=1,yr(i)?(e=!0,xr(n)):e=!1,ao(n,t),_o(n,i,r),Eo(n,i,r,t),$a(null,n,i,!0,e,t);case 19:return Xa(e,n,t)}throw Error(a(156,n.tag))};var Os=null,As=null;function Rs(e,n,t,i){this.tag=e,this.key=t,this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null,this.index=0,this.ref=null,this.pendingProps=n,this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null,this.mode=i,this.effectTag=0,this.lastEffect=this.firstEffect=this.nextEffect=null,this.childExpirationTime=this.expirationTime=0,this.alternate=null}function Ns(e,n,t,i){return new Rs(e,n,t,i)}function Is(e){return!(!(e=e.prototype)||!e.isReactComponent)}function Ls(e,n){var t=e.alternate;return null===t?((t=Ns(e.tag,n,e.key,e.mode)).elementType=e.elementType,t.type=e.type,t.stateNode=e.stateNode,t.alternate=e,e.alternate=t):(t.pendingProps=n,t.effectTag=0,t.nextEffect=null,t.firstEffect=null,t.lastEffect=null),t.childExpirationTime=e.childExpirationTime,t.expirationTime=e.expirationTime,t.child=e.child,t.memoizedProps=e.memoizedProps,t.memoizedState=e.memoizedState,t.updateQueue=e.updateQueue,n=e.dependencies,t.dependencies=null===n?null:{expirationTime:n.expirationTime,firstContext:n.firstContext,responders:n.responders},t.sibling=e.sibling,t.index=e.index,t.ref=e.ref,t}function Ds(e,n,t,i,r,o){var l=2;if(i=e,"function"==typeof e)Is(e)&&(l=1);else if("string"==typeof e)l=5;else e:switch(e){case te:return js(t.children,r,o,n);case le:l=8,r|=7;break;case ie:l=8,r|=1;break;case re:return(e=Ns(12,t,n,8|r)).elementType=re,e.type=re,e.expirationTime=o,e;case ce:return(e=Ns(13,t,n,r)).type=ce,e.elementType=ce,e.expirationTime=o,e;case ue:return(e=Ns(19,t,n,r)).elementType=ue,e.expirationTime=o,e;default:if("object"==typeof e&&null!==e)switch(e.$$typeof){case oe:l=10;break e;case ae:l=9;break e;case se:l=11;break e;case pe:l=14;break e;case de:l=16,i=null;break e;case fe:l=22;break e}throw Error(a(130,null==e?e:typeof e,""))}return(n=Ns(l,t,n,r)).elementType=e,n.type=i,n.expirationTime=o,n}function js(e,n,t,i){return(e=Ns(7,e,i,n)).expirationTime=t,e}function Ms(e,n,t){return(e=Ns(6,e,null,n)).expirationTime=t,e}function Fs(e,n,t){return(n=Ns(4,null!==e.children?e.children:[],e.key,n)).expirationTime=t,n.stateNode={containerInfo:e.containerInfo,pendingChildren:null,implementation:e.implementation},n}function Bs(e,n,t){this.tag=n,this.current=null,this.containerInfo=e,this.pingCache=this.pendingChildren=null,this.finishedExpirationTime=0,this.finishedWork=null,this.timeoutHandle=-1,this.pendingContext=this.context=null,this.hydrate=t,this.callbackNode=null,this.callbackPriority=90,this.lastExpiredTime=this.lastPingedTime=this.nextKnownPendingLevel=this.lastSuspendedTime=this.firstSuspendedTime=this.firstPendingTime=0}function zs(e,n){var t=e.firstSuspendedTime;return e=e.lastSuspendedTime,0!==t&&t>=n&&e<=n}function Us(e,n){var t=e.firstSuspendedTime,i=e.lastSuspendedTime;t<n&&(e.firstSuspendedTime=n),(i>n||0===t)&&(e.lastSuspendedTime=n),n<=e.lastPingedTime&&(e.lastPingedTime=0),n<=e.lastExpiredTime&&(e.lastExpiredTime=0)}function $s(e,n){n>e.firstPendingTime&&(e.firstPendingTime=n);var t=e.firstSuspendedTime;0!==t&&(n>=t?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:n>=e.lastSuspendedTime&&(e.lastSuspendedTime=n+1),n>e.nextKnownPendingLevel&&(e.nextKnownPendingLevel=n))}function Gs(e,n){var t=e.lastExpiredTime;(0===t||t>n)&&(e.lastExpiredTime=n)}function qs(e,n,t,i){var r=n.current,o=es(),l=vo.suspense;o=ns(o,r,l);e:if(t){n:{if(en(t=t._reactInternalFiber)!==t||1!==t.tag)throw Error(a(170));var s=t;do{switch(s.tag){case 3:s=s.stateNode.context;break n;case 1:if(yr(s.type)){s=s.stateNode.__reactInternalMemoizedMergedChildContext;break n}}s=s.return}while(null!==s);throw Error(a(171))}if(1===t.tag){var c=t.type;if(yr(c)){t=_r(t,c,s);break e}}t=s}else t=mr;return null===n.context?n.context=t:n.pendingContext=t,(n=po(o,l)).payload={element:e},null!==(i=void 0===i?null:i)&&(n.callback=i),fo(r,n),ts(r,o),o}function Hs(e){if(!(e=e.current).child)return null;switch(e.child.tag){case 5:default:return e.child.stateNode}}function Ws(e,n){null!==(e=e.memoizedState)&&null!==e.dehydrated&&e.retryTime<n&&(e.retryTime=n)}function Vs(e,n){Ws(e,n),(e=e.alternate)&&Ws(e,n)}function Ks(e,n,t){var i=new Bs(e,n,t=null!=t&&!0===t.hydrate),r=Ns(3,null,null,2===n?7:1===n?3:0);i.current=r,r.stateNode=i,co(r),e[At]=i.current,t&&0!==n&&function(e,n){var t=Ze(n);Cn.forEach((function(e){hn(e,n,t)})),On.forEach((function(e){hn(e,n,t)}))}(0,9===e.nodeType?e:e.ownerDocument),this._internalRoot=i}function Ys(e){return!(!e||1!==e.nodeType&&9!==e.nodeType&&11!==e.nodeType&&(8!==e.nodeType||" react-mount-point-unstable "!==e.nodeValue))}function Qs(e,n,t,i,r){var o=t._reactRootContainer;if(o){var a=o._internalRoot;if("function"==typeof r){var l=r;r=function(){var e=Hs(a);l.call(e)}}qs(n,a,e,r)}else{if(o=t._reactRootContainer=function(e,n){if(n||(n=!(!(n=e?9===e.nodeType?e.documentElement:e.firstChild:null)||1!==n.nodeType||!n.hasAttribute("data-reactroot"))),!n)for(var t;t=e.lastChild;)e.removeChild(t);return new Ks(e,0,n?{hydrate:!0}:void 0)}(t,i),a=o._internalRoot,"function"==typeof r){var s=r;r=function(){var e=Hs(a);s.call(e)}}cs((function(){qs(n,a,e,r)}))}return Hs(a)}function Xs(e,n,t){var i=3<arguments.length&&void 0!==arguments[3]?arguments[3]:null;return{$$typeof:ne,key:null==i?null:""+i,children:e,containerInfo:n,implementation:t}}function Js(e,n){var t=2<arguments.length&&void 0!==arguments[2]?arguments[2]:null;if(!Ys(n))throw Error(a(200));return Xs(e,n,null,t)}Ks.prototype.render=function(e){qs(e,this._internalRoot,null,null)},Ks.prototype.unmount=function(){var e=this._internalRoot,n=e.containerInfo;qs(null,e,null,(function(){n[At]=null}))},vn=function(e){if(13===e.tag){var n=Xr(es(),150,100);ts(e,n),Vs(e,n)}},bn=function(e){13===e.tag&&(ts(e,3),Vs(e,3))},yn=function(e){if(13===e.tag){var n=es();ts(e,n=ns(n,e,null)),Vs(e,n)}},C=function(e,n,t){switch(n){case"input":if(Se(e,t),n=t.name,"radio"===t.type&&null!=n){for(t=e;t.parentNode;)t=t.parentNode;for(t=t.querySelectorAll("input[name="+JSON.stringify(""+n)+'][type="radio"]'),n=0;n<t.length;n++){var i=t[n];if(i!==e&&i.form===e.form){var r=Lt(i);if(!r)throw Error(a(90));we(i),Se(i,r)}}}break;case"textarea":Ne(e,t);break;case"select":null!=(n=t.value)&&Oe(e,!!t.multiple,n,!1)}},L=ss,D=function(e,n,t,i,r){var o=Rl;Rl|=4;try{return Wr(98,e.bind(null,n,t,i,r))}finally{0===(Rl=o)&&Yr()}},j=function(){0==(49&Rl)&&(function(){if(null!==Ql){var e=Ql;Ql=null,e.forEach((function(e,n){Gs(n,e),os(n)})),Yr()}}(),xs())},M=function(e,n){var t=Rl;Rl|=2;try{return e(n)}finally{0===(Rl=t)&&Yr()}};var Zs={Events:[Nt,It,Lt,T,x,Ut,function(e){an(e,zt)},N,I,Zn,cn,xs,{current:!1}]};!function(e){var n=e.findFiberByHostInstance;(function(e){if("undefined"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__)return!1;var n=__REACT_DEVTOOLS_GLOBAL_HOOK__;if(n.isDisabled||!n.supportsFiber)return!0;try{var t=n.inject(e);Os=function(e){try{n.onCommitFiberRoot(t,e,void 0,64==(64&e.current.effectTag))}catch(i){}},As=function(e){try{n.onCommitFiberUnmount(t,e)}catch(i){}}}catch(i){}})(r({},e,{overrideHookState:null,overrideProps:null,setSuspenseHandler:null,scheduleUpdate:null,currentDispatcherRef:Q.ReactCurrentDispatcher,findHostInstanceByFiber:function(e){return null===(e=rn(e))?null:e.stateNode},findFiberByHostInstance:function(e){return n?n(e):null},findHostInstancesForRefresh:null,scheduleRefresh:null,scheduleRoot:null,setRefreshHandler:null,getCurrentFiber:null}))}({findFiberByHostInstance:Rt,bundleType:0,version:"16.14.0",rendererPackageName:"react-dom"}),n.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=Zs,n.createPortal=Js,n.findDOMNode=function(e){if(null==e)return null;if(1===e.nodeType)return e;var n=e._reactInternalFiber;if(void 0===n){if("function"==typeof e.render)throw Error(a(188));throw Error(a(268,Object.keys(e)))}return e=null===(e=rn(n))?null:e.stateNode},n.flushSync=function(e,n){if(0!=(48&Rl))throw Error(a(187));var t=Rl;Rl|=1;try{return Wr(99,e.bind(null,n))}finally{Rl=t,Yr()}},n.hydrate=function(e,n,t){if(!Ys(n))throw Error(a(200));return Qs(null,e,n,!0,t)},n.render=function(e,n,t){if(!Ys(n))throw Error(a(200));return Qs(null,e,n,!1,t)},n.unmountComponentAtNode=function(e){if(!Ys(e))throw Error(a(40));return!!e._reactRootContainer&&(cs((function(){Qs(null,null,e,!1,(function(){e._reactRootContainer=null,e[At]=null}))})),!0)},n.unstable_batchedUpdates=ss,n.unstable_createPortal=function(e,n){return Js(e,n,2<arguments.length&&void 0!==arguments[2]?arguments[2]:null)},n.unstable_renderSubtreeIntoContainer=function(e,n,t,i){if(!Ys(t))throw Error(a(200));if(null==e||void 0===e._reactInternalFiber)throw Error(a(38));return Qs(e,n,t,!1,i)},n.version="16.14.0"},function(e,n,t){"use strict";e.exports=t(42)},function(e,n,t){"use strict";var i,r,o,a,l;if("undefined"==typeof window||"function"!=typeof MessageChannel){var s=null,c=null,u=function(){if(null!==s)try{var e=n.unstable_now();s(!0,e),s=null}catch(t){throw setTimeout(u,0),t}},p=Date.now();n.unstable_now=function(){return Date.now()-p},i=function(e){null!==s?setTimeout(i,0,e):(s=e,setTimeout(u,0))},r=function(e,n){c=setTimeout(e,n)},o=function(){clearTimeout(c)},a=function(){return!1},l=n.unstable_forceFrameRate=function(){}}else{var d=window.performance,f=window.Date,m=window.setTimeout,g=window.clearTimeout;if("undefined"!=typeof console){var h=window.cancelAnimationFrame;"function"!=typeof window.requestAnimationFrame&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills"),"function"!=typeof h&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills")}if("object"==typeof d&&"function"==typeof d.now)n.unstable_now=function(){return d.now()};else{var v=f.now();n.unstable_now=function(){return f.now()-v}}var b=!1,y=null,k=-1,w=5,_=0;a=function(){return n.unstable_now()>=_},l=function(){},n.unstable_forceFrameRate=function(e){0>e||125<e?console.error("forceFrameRate takes a positive int between 0 and 125, forcing framerates higher than 125 fps is not unsupported"):w=0<e?Math.floor(1e3/e):5};var x=new MessageChannel,E=x.port2;x.port1.onmessage=function(){if(null!==y){var e=n.unstable_now();_=e+w;try{y(!0,e)?E.postMessage(null):(b=!1,y=null)}catch(t){throw E.postMessage(null),t}}else b=!1},i=function(e){y=e,b||(b=!0,E.postMessage(null))},r=function(e,t){k=m((function(){e(n.unstable_now())}),t)},o=function(){g(k),k=-1}}function S(e,n){var t=e.length;e.push(n);e:for(;;){var i=t-1>>>1,r=e[i];if(!(void 0!==r&&0<C(r,n)))break e;e[i]=n,e[t]=r,t=i}}function T(e){return void 0===(e=e[0])?null:e}function P(e){var n=e[0];if(void 0!==n){var t=e.pop();if(t!==n){e[0]=t;e:for(var i=0,r=e.length;i<r;){var o=2*(i+1)-1,a=e[o],l=o+1,s=e[l];if(void 0!==a&&0>C(a,t))void 0!==s&&0>C(s,a)?(e[i]=s,e[l]=t,i=l):(e[i]=a,e[o]=t,i=o);else{if(!(void 0!==s&&0>C(s,t)))break e;e[i]=s,e[l]=t,i=l}}}return n}return null}function C(e,n){var t=e.sortIndex-n.sortIndex;return 0!==t?t:e.id-n.id}var O=[],A=[],R=1,N=null,I=3,L=!1,D=!1,j=!1;function M(e){for(var n=T(A);null!==n;){if(null===n.callback)P(A);else{if(!(n.startTime<=e))break;P(A),n.sortIndex=n.expirationTime,S(O,n)}n=T(A)}}function F(e){if(j=!1,M(e),!D)if(null!==T(O))D=!0,i(B);else{var n=T(A);null!==n&&r(F,n.startTime-e)}}function B(e,t){D=!1,j&&(j=!1,o()),L=!0;var i=I;try{for(M(t),N=T(O);null!==N&&(!(N.expirationTime>t)||e&&!a());){var l=N.callback;if(null!==l){N.callback=null,I=N.priorityLevel;var s=l(N.expirationTime<=t);t=n.unstable_now(),"function"==typeof s?N.callback=s:N===T(O)&&P(O),M(t)}else P(O);N=T(O)}if(null!==N)var c=!0;else{var u=T(A);null!==u&&r(F,u.startTime-t),c=!1}return c}finally{N=null,I=i,L=!1}}function z(e){switch(e){case 1:return-1;case 2:return 250;case 5:return 1073741823;case 4:return 1e4;default:return 5e3}}var U=l;n.unstable_IdlePriority=5,n.unstable_ImmediatePriority=1,n.unstable_LowPriority=4,n.unstable_NormalPriority=3,n.unstable_Profiling=null,n.unstable_UserBlockingPriority=2,n.unstable_cancelCallback=function(e){e.callback=null},n.unstable_continueExecution=function(){D||L||(D=!0,i(B))},n.unstable_getCurrentPriorityLevel=function(){return I},n.unstable_getFirstCallbackNode=function(){return T(O)},n.unstable_next=function(e){switch(I){case 1:case 2:case 3:var n=3;break;default:n=I}var t=I;I=n;try{return e()}finally{I=t}},n.unstable_pauseExecution=function(){},n.unstable_requestPaint=U,n.unstable_runWithPriority=function(e,n){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var t=I;I=e;try{return n()}finally{I=t}},n.unstable_scheduleCallback=function(e,t,a){var l=n.unstable_now();if("object"==typeof a&&null!==a){var s=a.delay;s="number"==typeof s&&0<s?l+s:l,a="number"==typeof a.timeout?a.timeout:z(e)}else a=z(e),s=l;return e={id:R++,callback:t,priorityLevel:e,startTime:s,expirationTime:a=s+a,sortIndex:-1},s>l?(e.sortIndex=s,S(A,e),null===T(O)&&e===T(A)&&(j?o():j=!0,r(F,s-l))):(e.sortIndex=a,S(O,e),D||L||(D=!0,i(B))),e},n.unstable_shouldYield=function(){var e=n.unstable_now();M(e);var t=T(O);return t!==N&&null!==N&&null!==t&&null!==t.callback&&t.startTime<=e&&t.expirationTime<N.expirationTime||a()},n.unstable_wrapCallback=function(e){var n=I;return function(){var t=I;I=n;try{return e.apply(this,arguments)}finally{I=t}}}},function(e,n,t){"use strict";var i=t(44);function r(){}function o(){}o.resetWarningCache=r,e.exports=function(){function e(e,n,t,r,o,a){if(a!==i){var l=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw l.name="Invariant Violation",l}}function n(){return e}e.isRequired=e;var t={array:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:n,element:e,elementType:e,instanceOf:n,node:e,objectOf:n,oneOf:n,oneOfType:n,shape:n,exact:n,checkPropTypes:o,resetWarningCache:r};return t.PropTypes=t,t}},function(e,n,t){"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},function(e,n){e.exports=Array.isArray||function(e){return"[object Array]"==Object.prototype.toString.call(e)}},function(e,n,t){"use strict";var i="function"==typeof Symbol&&Symbol.for,r=i?Symbol.for("react.element"):60103,o=i?Symbol.for("react.portal"):60106,a=i?Symbol.for("react.fragment"):60107,l=i?Symbol.for("react.strict_mode"):60108,s=i?Symbol.for("react.profiler"):60114,c=i?Symbol.for("react.provider"):60109,u=i?Symbol.for("react.context"):60110,p=i?Symbol.for("react.async_mode"):60111,d=i?Symbol.for("react.concurrent_mode"):60111,f=i?Symbol.for("react.forward_ref"):60112,m=i?Symbol.for("react.suspense"):60113,g=i?Symbol.for("react.suspense_list"):60120,h=i?Symbol.for("react.memo"):60115,v=i?Symbol.for("react.lazy"):60116,b=i?Symbol.for("react.block"):60121,y=i?Symbol.for("react.fundamental"):60117,k=i?Symbol.for("react.responder"):60118,w=i?Symbol.for("react.scope"):60119;function _(e){if("object"==typeof e&&null!==e){var n=e.$$typeof;switch(n){case r:switch(e=e.type){case p:case d:case a:case s:case l:case m:return e;default:switch(e=e&&e.$$typeof){case u:case f:case v:case h:case c:return e;default:return n}}case o:return n}}}function x(e){return _(e)===d}n.AsyncMode=p,n.ConcurrentMode=d,n.ContextConsumer=u,n.ContextProvider=c,n.Element=r,n.ForwardRef=f,n.Fragment=a,n.Lazy=v,n.Memo=h,n.Portal=o,n.Profiler=s,n.StrictMode=l,n.Suspense=m,n.isAsyncMode=function(e){return x(e)||_(e)===p},n.isConcurrentMode=x,n.isContextConsumer=function(e){return _(e)===u},n.isContextProvider=function(e){return _(e)===c},n.isElement=function(e){return"object"==typeof e&&null!==e&&e.$$typeof===r},n.isForwardRef=function(e){return _(e)===f},n.isFragment=function(e){return _(e)===a},n.isLazy=function(e){return _(e)===v},n.isMemo=function(e){return _(e)===h},n.isPortal=function(e){return _(e)===o},n.isProfiler=function(e){return _(e)===s},n.isStrictMode=function(e){return _(e)===l},n.isSuspense=function(e){return _(e)===m},n.isValidElementType=function(e){return"string"==typeof e||"function"==typeof e||e===a||e===d||e===s||e===l||e===m||e===g||"object"==typeof e&&null!==e&&(e.$$typeof===v||e.$$typeof===h||e.$$typeof===c||e.$$typeof===u||e.$$typeof===f||e.$$typeof===y||e.$$typeof===k||e.$$typeof===w||e.$$typeof===b)},n.typeOf=_},,function(e,n,t){const i=t(49),r=t(50),o=new Set;function a(e){void 0===e?e=Object.keys(i.languages).filter((e=>"meta"!=e)):Array.isArray(e)||(e=[e]);const n=[...o,...Object.keys(Prism.languages)];r(i,e,n).load((e=>{if(!(e in i.languages))return void(a.silent||console.warn("Language does not exist: "+e));const n="./prism-"+e;delete t.c[t(19).resolve(n)],delete Prism.languages[e],t(19)(n),o.add(e)}))}a.silent=!1,e.exports=a},function(e,n,t){e.exports&&(e.exports={core:{meta:{path:"components/prism-core.js",option:"mandatory"},core:"Core"},themes:{meta:{path:"themes/{id}.css",link:"index.html?theme={id}",exclusive:!0},prism:{title:"Default",option:"default"},"prism-dark":"Dark","prism-funky":"Funky","prism-okaidia":{title:"Okaidia",owner:"ocodia"},"prism-twilight":{title:"Twilight",owner:"remybach"},"prism-coy":{title:"Coy",owner:"tshedor"},"prism-solarizedlight":{title:"Solarized Light",owner:"hectormatos2011 "},"prism-tomorrow":{title:"Tomorrow Night",owner:"Rosey"}},languages:{meta:{path:"components/prism-{id}",noCSS:!0,examplesPath:"examples/prism-{id}",addCheckAll:!0},markup:{title:"Markup",alias:["html","xml","svg","mathml","ssml","atom","rss"],aliasTitles:{html:"HTML",xml:"XML",svg:"SVG",mathml:"MathML",ssml:"SSML",atom:"Atom",rss:"RSS"},option:"default"},css:{title:"CSS",option:"default",modify:"markup"},clike:{title:"C-like",option:"default"},javascript:{title:"JavaScript",require:"clike",modify:"markup",optional:"regex",alias:"js",option:"default"},abap:{title:"ABAP",owner:"dellagustin"},abnf:{title:"ABNF",owner:"RunDevelopment"},actionscript:{title:"ActionScript",require:"javascript",modify:"markup",owner:"Golmote"},ada:{title:"Ada",owner:"Lucretia"},agda:{title:"Agda",owner:"xy-ren"},al:{title:"AL",owner:"RunDevelopment"},antlr4:{title:"ANTLR4",alias:"g4",owner:"RunDevelopment"},apacheconf:{title:"Apache Configuration",owner:"GuiTeK"},apex:{title:"Apex",require:["clike","sql"],owner:"RunDevelopment"},apl:{title:"APL",owner:"ngn"},applescript:{title:"AppleScript",owner:"Golmote"},aql:{title:"AQL",owner:"RunDevelopment"},arduino:{title:"Arduino",require:"cpp",owner:"dkern"},arff:{title:"ARFF",owner:"Golmote"},asciidoc:{alias:"adoc",title:"AsciiDoc",owner:"Golmote"},aspnet:{title:"ASP.NET (C#)",require:["markup","csharp"],owner:"nauzilus"},asm6502:{title:"6502 Assembly",owner:"kzurawel"},autohotkey:{title:"AutoHotkey",owner:"aviaryan"},autoit:{title:"AutoIt",owner:"Golmote"},avisynth:{title:"AviSynth",alias:"avs",owner:"Zinfidel"},"avro-idl":{title:"Avro IDL",alias:"avdl",owner:"RunDevelopment"},bash:{title:"Bash",alias:"shell",aliasTitles:{shell:"Shell"},owner:"zeitgeist87"},basic:{title:"BASIC",owner:"Golmote"},batch:{title:"Batch",owner:"Golmote"},bbcode:{title:"BBcode",alias:"shortcode",aliasTitles:{shortcode:"Shortcode"},owner:"RunDevelopment"},bicep:{title:"Bicep",owner:"johnnyreilly"},birb:{title:"Birb",require:"clike",owner:"Calamity210"},bison:{title:"Bison",require:"c",owner:"Golmote"},bnf:{title:"BNF",alias:"rbnf",aliasTitles:{rbnf:"RBNF"},owner:"RunDevelopment"},brainfuck:{title:"Brainfuck",owner:"Golmote"},brightscript:{title:"BrightScript",owner:"RunDevelopment"},bro:{title:"Bro",owner:"wayward710"},bsl:{title:"BSL (1C:Enterprise)",alias:"oscript",aliasTitles:{oscript:"OneScript"},owner:"Diversus23"},c:{title:"C",require:"clike",owner:"zeitgeist87"},csharp:{title:"C#",require:"clike",alias:["cs","dotnet"],owner:"mvalipour"},cpp:{title:"C++",require:"c",owner:"zeitgeist87"},cfscript:{title:"CFScript",require:"clike",alias:"cfc",owner:"mjclemente"},chaiscript:{title:"ChaiScript",require:["clike","cpp"],owner:"RunDevelopment"},cil:{title:"CIL",owner:"sbrl"},clojure:{title:"Clojure",owner:"troglotit"},cmake:{title:"CMake",owner:"mjrogozinski"},cobol:{title:"COBOL",owner:"RunDevelopment"},coffeescript:{title:"CoffeeScript",require:"javascript",alias:"coffee",owner:"R-osey"},concurnas:{title:"Concurnas",alias:"conc",owner:"jasontatton"},csp:{title:"Content-Security-Policy",owner:"ScottHelme"},coq:{title:"Coq",owner:"RunDevelopment"},crystal:{title:"Crystal",require:"ruby",owner:"MakeNowJust"},"css-extras":{title:"CSS Extras",require:"css",modify:"css",owner:"milesj"},csv:{title:"CSV",owner:"RunDevelopment"},cypher:{title:"Cypher",owner:"RunDevelopment"},d:{title:"D",require:"clike",owner:"Golmote"},dart:{title:"Dart",require:"clike",owner:"Golmote"},dataweave:{title:"DataWeave",owner:"machaval"},dax:{title:"DAX",owner:"peterbud"},dhall:{title:"Dhall",owner:"RunDevelopment"},diff:{title:"Diff",owner:"uranusjr"},django:{title:"Django/Jinja2",require:"markup-templating",alias:"jinja2",owner:"romanvm"},"dns-zone-file":{title:"DNS zone file",owner:"RunDevelopment",alias:"dns-zone"},docker:{title:"Docker",alias:"dockerfile",owner:"JustinBeckwith"},dot:{title:"DOT (Graphviz)",alias:"gv",optional:"markup",owner:"RunDevelopment"},ebnf:{title:"EBNF",owner:"RunDevelopment"},editorconfig:{title:"EditorConfig",owner:"osipxd"},eiffel:{title:"Eiffel",owner:"Conaclos"},ejs:{title:"EJS",require:["javascript","markup-templating"],owner:"RunDevelopment",alias:"eta",aliasTitles:{eta:"Eta"}},elixir:{title:"Elixir",owner:"Golmote"},elm:{title:"Elm",owner:"zwilias"},etlua:{title:"Embedded Lua templating",require:["lua","markup-templating"],owner:"RunDevelopment"},erb:{title:"ERB",require:["ruby","markup-templating"],owner:"Golmote"},erlang:{title:"Erlang",owner:"Golmote"},"excel-formula":{title:"Excel Formula",alias:["xlsx","xls"],owner:"RunDevelopment"},fsharp:{title:"F#",require:"clike",owner:"simonreynolds7"},factor:{title:"Factor",owner:"catb0t"},false:{title:"False",owner:"edukisto"},"firestore-security-rules":{title:"Firestore security rules",require:"clike",owner:"RunDevelopment"},flow:{title:"Flow",require:"javascript",owner:"Golmote"},fortran:{title:"Fortran",owner:"Golmote"},ftl:{title:"FreeMarker Template Language",require:"markup-templating",owner:"RunDevelopment"},gml:{title:"GameMaker Language",alias:"gamemakerlanguage",require:"clike",owner:"LiarOnce"},gap:{title:"GAP (CAS)",owner:"RunDevelopment"},gcode:{title:"G-code",owner:"RunDevelopment"},gdscript:{title:"GDScript",owner:"RunDevelopment"},gedcom:{title:"GEDCOM",owner:"Golmote"},gherkin:{title:"Gherkin",owner:"hason"},git:{title:"Git",owner:"lgiraudel"},glsl:{title:"GLSL",require:"c",owner:"Golmote"},gn:{title:"GN",alias:"gni",owner:"RunDevelopment"},go:{title:"Go",require:"clike",owner:"arnehormann"},graphql:{title:"GraphQL",optional:"markdown",owner:"Golmote"},groovy:{title:"Groovy",require:"clike",owner:"robfletcher"},haml:{title:"Haml",require:"ruby",optional:["css","css-extras","coffeescript","erb","javascript","less","markdown","scss","textile"],owner:"Golmote"},handlebars:{title:"Handlebars",require:"markup-templating",alias:"hbs",owner:"Golmote"},haskell:{title:"Haskell",alias:"hs",owner:"bholst"},haxe:{title:"Haxe",require:"clike",owner:"Golmote"},hcl:{title:"HCL",owner:"outsideris"},hlsl:{title:"HLSL",require:"c",owner:"RunDevelopment"},hoon:{title:"Hoon",owner:"matildepark"},http:{title:"HTTP",optional:["css","javascript","json","markup","uri"],owner:"danielgtaylor"},hpkp:{title:"HTTP Public-Key-Pins",owner:"ScottHelme"},hsts:{title:"HTTP Strict-Transport-Security",owner:"ScottHelme"},ichigojam:{title:"IchigoJam",owner:"BlueCocoa"},icon:{title:"Icon",owner:"Golmote"},"icu-message-format":{title:"ICU Message Format",owner:"RunDevelopment"},idris:{title:"Idris",alias:"idr",owner:"KeenS",require:"haskell"},ignore:{title:".ignore",owner:"osipxd",alias:["gitignore","hgignore","npmignore"],aliasTitles:{gitignore:".gitignore",hgignore:".hgignore",npmignore:".npmignore"}},inform7:{title:"Inform 7",owner:"Golmote"},ini:{title:"Ini",owner:"aviaryan"},io:{title:"Io",owner:"AlesTsurko"},j:{title:"J",owner:"Golmote"},java:{title:"Java",require:"clike",owner:"sherblot"},javadoc:{title:"JavaDoc",require:["markup","java","javadoclike"],modify:"java",optional:"scala",owner:"RunDevelopment"},javadoclike:{title:"JavaDoc-like",modify:["java","javascript","php"],owner:"RunDevelopment"},javastacktrace:{title:"Java stack trace",owner:"RunDevelopment"},jexl:{title:"Jexl",owner:"czosel"},jolie:{title:"Jolie",require:"clike",owner:"thesave"},jq:{title:"JQ",owner:"RunDevelopment"},jsdoc:{title:"JSDoc",require:["javascript","javadoclike","typescript"],modify:"javascript",optional:["actionscript","coffeescript"],owner:"RunDevelopment"},"js-extras":{title:"JS Extras",require:"javascript",modify:"javascript",optional:["actionscript","coffeescript","flow","n4js","typescript"],owner:"RunDevelopment"},json:{title:"JSON",alias:"webmanifest",aliasTitles:{webmanifest:"Web App Manifest"},owner:"CupOfTea696"},json5:{title:"JSON5",require:"json",owner:"RunDevelopment"},jsonp:{title:"JSONP",require:"json",owner:"RunDevelopment"},jsstacktrace:{title:"JS stack trace",owner:"sbrl"},"js-templates":{title:"JS Templates",require:"javascript",modify:"javascript",optional:["css","css-extras","graphql","markdown","markup","sql"],owner:"RunDevelopment"},julia:{title:"Julia",owner:"cdagnino"},keyman:{title:"Keyman",owner:"mcdurdin"},kotlin:{title:"Kotlin",alias:["kt","kts"],aliasTitles:{kts:"Kotlin Script"},require:"clike",owner:"Golmote"},kumir:{title:"KuMir (\u041a\u0443\u041c\u0438\u0440)",alias:"kum",owner:"edukisto"},kusto:{title:"Kusto",owner:"RunDevelopment"},latex:{title:"LaTeX",alias:["tex","context"],aliasTitles:{tex:"TeX",context:"ConTeXt"},owner:"japborst"},latte:{title:"Latte",require:["clike","markup-templating","php"],owner:"nette"},less:{title:"Less",require:"css",optional:"css-extras",owner:"Golmote"},lilypond:{title:"LilyPond",require:"scheme",alias:"ly",owner:"RunDevelopment"},liquid:{title:"Liquid",require:"markup-templating",owner:"cinhtau"},lisp:{title:"Lisp",alias:["emacs","elisp","emacs-lisp"],owner:"JuanCaicedo"},livescript:{title:"LiveScript",owner:"Golmote"},llvm:{title:"LLVM IR",owner:"porglezomp"},log:{title:"Log file",optional:"javastacktrace",owner:"RunDevelopment"},lolcode:{title:"LOLCODE",owner:"Golmote"},lua:{title:"Lua",owner:"Golmote"},magma:{title:"Magma (CAS)",owner:"RunDevelopment"},makefile:{title:"Makefile",owner:"Golmote"},markdown:{title:"Markdown",require:"markup",optional:"yaml",alias:"md",owner:"Golmote"},"markup-templating":{title:"Markup templating",require:"markup",owner:"Golmote"},matlab:{title:"MATLAB",owner:"Golmote"},maxscript:{title:"MAXScript",owner:"RunDevelopment"},mel:{title:"MEL",owner:"Golmote"},mermaid:{title:"Mermaid",owner:"RunDevelopment"},mizar:{title:"Mizar",owner:"Golmote"},mongodb:{title:"MongoDB",owner:"airs0urce",require:"javascript"},monkey:{title:"Monkey",owner:"Golmote"},moonscript:{title:"MoonScript",alias:"moon",owner:"RunDevelopment"},n1ql:{title:"N1QL",owner:"TMWilds"},n4js:{title:"N4JS",require:"javascript",optional:"jsdoc",alias:"n4jsd",owner:"bsmith-n4"},"nand2tetris-hdl":{title:"Nand To Tetris HDL",owner:"stephanmax"},naniscript:{title:"Naninovel Script",owner:"Elringus",alias:"nani"},nasm:{title:"NASM",owner:"rbmj"},neon:{title:"NEON",owner:"nette"},nevod:{title:"Nevod",owner:"nezaboodka"},nginx:{title:"nginx",owner:"volado"},nim:{title:"Nim",owner:"Golmote"},nix:{title:"Nix",owner:"Golmote"},nsis:{title:"NSIS",owner:"idleberg"},objectivec:{title:"Objective-C",require:"c",alias:"objc",owner:"uranusjr"},ocaml:{title:"OCaml",owner:"Golmote"},opencl:{title:"OpenCL",require:"c",modify:["c","cpp"],owner:"Milania1"},openqasm:{title:"OpenQasm",alias:"qasm",owner:"RunDevelopment"},oz:{title:"Oz",owner:"Golmote"},parigp:{title:"PARI/GP",owner:"Golmote"},parser:{title:"Parser",require:"markup",owner:"Golmote"},pascal:{title:"Pascal",alias:"objectpascal",aliasTitles:{objectpascal:"Object Pascal"},owner:"Golmote"},pascaligo:{title:"Pascaligo",owner:"DefinitelyNotAGoat"},psl:{title:"PATROL Scripting Language",owner:"bertysentry"},pcaxis:{title:"PC-Axis",alias:"px",owner:"RunDevelopment"},peoplecode:{title:"PeopleCode",alias:"pcode",owner:"RunDevelopment"},perl:{title:"Perl",owner:"Golmote"},php:{title:"PHP",require:"markup-templating",owner:"milesj"},phpdoc:{title:"PHPDoc",require:["php","javadoclike"],modify:"php",owner:"RunDevelopment"},"php-extras":{title:"PHP Extras",require:"php",modify:"php",owner:"milesj"},plsql:{title:"PL/SQL",require:"sql",owner:"Golmote"},powerquery:{title:"PowerQuery",alias:["pq","mscript"],owner:"peterbud"},powershell:{title:"PowerShell",owner:"nauzilus"},processing:{title:"Processing",require:"clike",owner:"Golmote"},prolog:{title:"Prolog",owner:"Golmote"},promql:{title:"PromQL",owner:"arendjr"},properties:{title:".properties",owner:"Golmote"},protobuf:{title:"Protocol Buffers",require:"clike",owner:"just-boris"},pug:{title:"Pug",require:["markup","javascript"],optional:["coffeescript","ejs","handlebars","less","livescript","markdown","scss","stylus","twig"],owner:"Golmote"},puppet:{title:"Puppet",owner:"Golmote"},pure:{title:"Pure",optional:["c","cpp","fortran"],owner:"Golmote"},purebasic:{title:"PureBasic",require:"clike",alias:"pbfasm",owner:"HeX0R101"},purescript:{title:"PureScript",require:"haskell",alias:"purs",owner:"sriharshachilakapati"},python:{title:"Python",alias:"py",owner:"multipetros"},qsharp:{title:"Q#",require:"clike",alias:"qs",owner:"fedonman"},q:{title:"Q (kdb+ database)",owner:"Golmote"},qml:{title:"QML",require:"javascript",owner:"RunDevelopment"},qore:{title:"Qore",require:"clike",owner:"temnroegg"},r:{title:"R",owner:"Golmote"},racket:{title:"Racket",require:"scheme",alias:"rkt",owner:"RunDevelopment"},cshtml:{title:"Razor C#",alias:"razor",require:["markup","csharp"],optional:["css","css-extras","javascript","js-extras"],owner:"RunDevelopment"},jsx:{title:"React JSX",require:["markup","javascript"],optional:["jsdoc","js-extras","js-templates"],owner:"vkbansal"},tsx:{title:"React TSX",require:["jsx","typescript"]},reason:{title:"Reason",require:"clike",owner:"Golmote"},regex:{title:"Regex",owner:"RunDevelopment"},rego:{title:"Rego",owner:"JordanSh"},renpy:{title:"Ren'py",alias:"rpy",owner:"HyuchiaDiego"},rest:{title:"reST (reStructuredText)",owner:"Golmote"},rip:{title:"Rip",owner:"ravinggenius"},roboconf:{title:"Roboconf",owner:"Golmote"},robotframework:{title:"Robot Framework",alias:"robot",owner:"RunDevelopment"},ruby:{title:"Ruby",require:"clike",alias:"rb",owner:"samflores"},rust:{title:"Rust",owner:"Golmote"},sas:{title:"SAS",optional:["groovy","lua","sql"],owner:"Golmote"},sass:{title:"Sass (Sass)",require:"css",optional:"css-extras",owner:"Golmote"},scss:{title:"Sass (Scss)",require:"css",optional:"css-extras",owner:"MoOx"},scala:{title:"Scala",require:"java",owner:"jozic"},scheme:{title:"Scheme",owner:"bacchus123"},"shell-session":{title:"Shell session",require:"bash",alias:["sh-session","shellsession"],owner:"RunDevelopment"},smali:{title:"Smali",owner:"RunDevelopment"},smalltalk:{title:"Smalltalk",owner:"Golmote"},smarty:{title:"Smarty",require:"markup-templating",owner:"Golmote"},sml:{title:"SML",alias:"smlnj",aliasTitles:{smlnj:"SML/NJ"},owner:"RunDevelopment"},solidity:{title:"Solidity (Ethereum)",alias:"sol",require:"clike",owner:"glachaud"},"solution-file":{title:"Solution file",alias:"sln",owner:"RunDevelopment"},soy:{title:"Soy (Closure Template)",require:"markup-templating",owner:"Golmote"},sparql:{title:"SPARQL",require:"turtle",owner:"Triply-Dev",alias:"rq"},"splunk-spl":{title:"Splunk SPL",owner:"RunDevelopment"},sqf:{title:"SQF: Status Quo Function (Arma 3)",require:"clike",owner:"RunDevelopment"},sql:{title:"SQL",owner:"multipetros"},squirrel:{title:"Squirrel",require:"clike",owner:"RunDevelopment"},stan:{title:"Stan",owner:"RunDevelopment"},iecst:{title:"Structured Text (IEC 61131-3)",owner:"serhioromano"},stylus:{title:"Stylus",owner:"vkbansal"},swift:{title:"Swift",owner:"chrischares"},systemd:{title:"Systemd configuration file",owner:"RunDevelopment"},"t4-templating":{title:"T4 templating",owner:"RunDevelopment"},"t4-cs":{title:"T4 Text Templates (C#)",require:["t4-templating","csharp"],alias:"t4",owner:"RunDevelopment"},"t4-vb":{title:"T4 Text Templates (VB)",require:["t4-templating","vbnet"],owner:"RunDevelopment"},tap:{title:"TAP",owner:"isaacs",require:"yaml"},tcl:{title:"Tcl",owner:"PeterChaplin"},tt2:{title:"Template Toolkit 2",require:["clike","markup-templating"],owner:"gflohr"},textile:{title:"Textile",require:"markup",optional:"css",owner:"Golmote"},toml:{title:"TOML",owner:"RunDevelopment"},turtle:{title:"Turtle",alias:"trig",aliasTitles:{trig:"TriG"},owner:"jakubklimek"},twig:{title:"Twig",require:"markup",owner:"brandonkelly"},typescript:{title:"TypeScript",require:"javascript",optional:"js-templates",alias:"ts",owner:"vkbansal"},typoscript:{title:"TypoScript",alias:"tsconfig",aliasTitles:{tsconfig:"TSConfig"},owner:"dkern"},unrealscript:{title:"UnrealScript",alias:["uscript","uc"],owner:"RunDevelopment"},uri:{title:"URI",alias:"url",aliasTitles:{url:"URL"},owner:"RunDevelopment"},v:{title:"V",require:"clike",owner:"taggon"},vala:{title:"Vala",require:"clike",optional:"regex",owner:"TemplarVolk"},vbnet:{title:"VB.Net",require:"basic",owner:"Bigsby"},velocity:{title:"Velocity",require:"markup",owner:"Golmote"},verilog:{title:"Verilog",owner:"a-rey"},vhdl:{title:"VHDL",owner:"a-rey"},vim:{title:"vim",owner:"westonganger"},"visual-basic":{title:"Visual Basic",alias:["vb","vba"],aliasTitles:{vba:"VBA"},owner:"Golmote"},warpscript:{title:"WarpScript",owner:"RunDevelopment"},wasm:{title:"WebAssembly",owner:"Golmote"},wiki:{title:"Wiki markup",require:"markup",owner:"Golmote"},wolfram:{title:"Wolfram language",alias:["mathematica","nb","wl"],aliasTitles:{mathematica:"Mathematica",nb:"Mathematica Notebook"},owner:"msollami"},wren:{title:"Wren",owner:"clsource"},xeora:{title:"Xeora",require:"markup",alias:"xeoracube",aliasTitles:{xeoracube:"XeoraCube"},owner:"freakmaxi"},"xml-doc":{title:"XML doc (.net)",require:"markup",modify:["csharp","fsharp","vbnet"],owner:"RunDevelopment"},xojo:{title:"Xojo (REALbasic)",owner:"Golmote"},xquery:{title:"XQuery",require:"markup",owner:"Golmote"},yaml:{title:"YAML",alias:"yml",owner:"hason"},yang:{title:"YANG",owner:"RunDevelopment"},zig:{title:"Zig",owner:"RunDevelopment"}},plugins:{meta:{path:"plugins/{id}/prism-{id}",link:"plugins/{id}/"},"line-highlight":{title:"Line Highlight",description:"Highlights specific lines and/or line ranges."},"line-numbers":{title:"Line Numbers",description:"Line number at the beginning of code lines.",owner:"kuba-kubula"},"show-invisibles":{title:"Show Invisibles",description:"Show hidden characters such as tabs and line breaks.",optional:["autolinker","data-uri-highlight"]},autolinker:{title:"Autolinker",description:"Converts URLs and emails in code to clickable links. Parses Markdown links in comments."},wpd:{title:"WebPlatform Docs",description:'Makes tokens link to <a href="https://webplatform.github.io/docs/">WebPlatform.org documentation</a>. The links open in a new tab.'},"custom-class":{title:"Custom Class",description:"This plugin allows you to prefix Prism's default classes (<code>.comment</code> can become <code>.namespace--comment</code>) or replace them with your defined ones (like <code>.editor__comment</code>). You can even add new classes.",owner:"dvkndn",noCSS:!0},"file-highlight":{title:"File Highlight",description:"Fetch external files and highlight them with Prism. Used on the Prism website itself.",noCSS:!0},"show-language":{title:"Show Language",description:"Display the highlighted language in code blocks (inline code does not show the label).",owner:"nauzilus",noCSS:!0,require:"toolbar"},"jsonp-highlight":{title:"JSONP Highlight",description:"Fetch content with JSONP and highlight some interesting content (e.g. GitHub/Gists or Bitbucket API).",noCSS:!0,owner:"nauzilus"},"highlight-keywords":{title:"Highlight Keywords",description:"Adds special CSS classes for each keyword for fine-grained highlighting.",owner:"vkbansal",noCSS:!0},"remove-initial-line-feed":{title:"Remove initial line feed",description:"Removes the initial line feed in code blocks.",owner:"Golmote",noCSS:!0},"inline-color":{title:"Inline color",description:"Adds a small inline preview for colors in style sheets.",require:"css-extras",owner:"RunDevelopment"},previewers:{title:"Previewers",description:"Previewers for angles, colors, gradients, easing and time.",require:"css-extras",owner:"Golmote"},autoloader:{title:"Autoloader",description:"Automatically loads the needed languages to highlight the code blocks.",owner:"Golmote",noCSS:!0},"keep-markup":{title:"Keep Markup",description:"Prevents custom markup from being dropped out during highlighting.",owner:"Golmote",optional:"normalize-whitespace",noCSS:!0},"command-line":{title:"Command Line",description:"Display a command line with a prompt and, optionally, the output/response from the commands.",owner:"chriswells0"},"unescaped-markup":{title:"Unescaped Markup",description:"Write markup without having to escape anything."},"normalize-whitespace":{title:"Normalize Whitespace",description:"Supports multiple operations to normalize whitespace in code blocks.",owner:"zeitgeist87",optional:"unescaped-markup",noCSS:!0},"data-uri-highlight":{title:"Data-URI Highlight",description:"Highlights data-URI contents.",owner:"Golmote",noCSS:!0},toolbar:{title:"Toolbar",description:"Attach a toolbar for plugins to easily register buttons on the top of a code block.",owner:"mAAdhaTTah"},"copy-to-clipboard":{title:"Copy to Clipboard Button",description:"Add a button that copies the code block to the clipboard when clicked.",owner:"mAAdhaTTah",require:"toolbar",noCSS:!0},"download-button":{title:"Download Button",description:"A button in the toolbar of a code block adding a convenient way to download a code file.",owner:"Golmote",require:"toolbar",noCSS:!0},"match-braces":{title:"Match braces",description:"Highlights matching braces.",owner:"RunDevelopment"},"diff-highlight":{title:"Diff Highlight",description:"Highlights the code inside diff blocks.",owner:"RunDevelopment",require:"diff"},"filter-highlight-all":{title:"Filter highlightAll",description:"Filters the elements the <code>highlightAll</code> and <code>highlightAllUnder</code> methods actually highlight.",owner:"RunDevelopment",noCSS:!0},treeview:{title:"Treeview",description:"A language with special styles to highlight file system tree structures.",owner:"Golmote"}}})},function(e,n,t){"use strict";var i=function(){var e=function(){};function n(e,n){Array.isArray(e)?e.forEach(n):null!=e&&n(e,0)}function t(e){for(var n={},t=0,i=e.length;t<i;t++)n[e[t]]=!0;return n}function i(e){var t={},i=[];function r(i,o){if(!(i in t)){o.push(i);var a=o.indexOf(i);if(a<o.length-1)throw new Error("Circular dependency: "+o.slice(a).join(" -> "));var l={},s=e[i];if(s){function c(n){if(!(n in e))throw new Error(i+" depends on an unknown component "+n);if(!(n in l))for(var a in r(n,o),l[n]=!0,t[n])l[a]=!0}n(s.require,c),n(s.optional,c),n(s.modify,c)}t[i]=l,o.pop()}}return function(e){var n=t[e];return n||(r(e,i),n=t[e]),n}}function r(e){for(var n in e)return!0;return!1}return function(o,a,l){var s=function(e){var n={};for(var t in e){var i=e[t];for(var r in i)if("meta"!=r){var o=i[r];n[r]="string"==typeof o?{title:o}:o}}return n}(o),c=function(e){var t;return function(i){if(i in e)return i;if(!t)for(var r in t={},e){var o=e[r];n(o&&o.alias,(function(n){if(n in t)throw new Error(n+" cannot be alias for both "+r+" and "+t[n]);if(n in e)throw new Error(n+" cannot be alias of "+r+" because it is a component.");t[n]=r}))}return t[i]||i}}(s);a=a.map(c),l=(l||[]).map(c);var u=t(a),p=t(l);a.forEach((function e(t){var i=s[t];n(i&&i.require,(function(n){n in p||(u[n]=!0,e(n))}))}));for(var d,f=i(s),m=u;r(m);){for(var g in d={},m){var h=s[g];n(h&&h.modify,(function(e){e in p&&(d[e]=!0)}))}for(var v in p)if(!(v in u))for(var b in f(v))if(b in u){d[v]=!0;break}for(var y in m=d)u[y]=!0}var k={getIds:function(){var e=[];return k.load((function(n){e.push(n)})),e},load:function(n,t){return function(n,t,i,r){var o=r?r.series:void 0,a=r?r.parallel:e,l={},s={};function c(e){if(e in l)return l[e];s[e]=!0;var r,u=[];for(var p in n(e))p in t&&u.push(p);if(0===u.length)r=i(e);else{var d=a(u.map((function(e){var n=c(e);return delete s[e],n})));o?r=o(d,(function(){return i(e)})):i(e)}return l[e]=r}for(var u in t)c(u);var p=[];for(var d in s)p.push(l[d]);return a(p)}(f,u,n,t)}};return k}}();e.exports=i},,,,function(e,n,t){"use strict";t.r(n);var i=t(0),r=t.n(i),o=t(30),a=t(10),l=t(20),s=t.n(l),c=function(e){var n=e.error,t=e.retry,i=e.pastDelay;return n?r.a.createElement("div",{style:{align:"center",color:"#fff",backgroundColor:"#fa383e",borderColor:"#fa383e",borderStyle:"solid",borderRadius:"0.25rem",borderWidth:"1px",boxSizing:"border-box",display:"block",padding:"1rem",flex:"0 0 50%",marginLeft:"25%",marginRight:"25%",marginTop:"5rem",maxWidth:"50%",width:"100%"}},r.a.createElement("p",null,n.message),r.a.createElement("div",null,r.a.createElement("button",{type:"button",onClick:t},"Retry"))):i?r.a.createElement("div",{style:{display:"flex",justifyContent:"center",alignItems:"center",height:"100vh"}},r.a.createElement("svg",{id:"loader",style:{width:128,height:110,position:"absolute",top:"calc(100vh - 64%)"},viewBox:"0 0 45 45",xmlns:"http://www.w3.org/2000/svg",stroke:"#61dafb"},r.a.createElement("g",{fill:"none",fillRule:"evenodd",transform:"translate(1 1)",strokeWidth:"2"},r.a.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.a.createElement("animate",{attributeName:"r",begin:"1.5s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-opacity",begin:"1.5s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-width",begin:"1.5s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.a.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.a.createElement("animate",{attributeName:"r",begin:"3s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-opacity",begin:"3s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-width",begin:"3s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.a.createElement("circle",{cx:"22",cy:"22",r:"8"},r.a.createElement("animate",{attributeName:"r",begin:"0s",dur:"1.5s",values:"6;1;2;3;4;5;6",calcMode:"linear",repeatCount:"indefinite"}))))):null},u=t(13),p={"012f72bc":[function(){return t.e(3).then(t.bind(null,65))},"@site/versioned_docs/version-1.2.11/pipeline/logstash.md",65],"0214960a":[function(){return t.e(4).then(t.bind(null,66))},"@site/versioned_docs/version-1.2.8/devel/docker.md",66],"02341ef2":[function(){return t.e(5).then(t.t.bind(null,67,3))},"~docs/default/version-1-2-11-metadata-prop-386.json",67],"0271802f":[function(){return t.e(6).then(t.bind(null,68))},"@site/versioned_docs/version-1.2.10/pipeline/logstash.md",68],"0327c8bf":[function(){return t.e(7).then(t.bind(null,69))},"@site/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",69],"0381afdd":[function(){return t.e(8).then(t.bind(null,70))},"@site/versioned_docs/version-1.2.7/documentation_guide.md",70],"05c6fd0e":[function(){return t.e(9).then(t.bind(null,71))},"@site/versioned_docs/version-1.2.11/deploy/choosing.md",71],"066f8364":[function(){return t.e(10).then(t.bind(null,72))},"@site/versioned_docs/version-1.2.5/pipeline_importer.md",72],"07faddd5":[function(){return t.e(11).then(t.bind(null,73))},"@site/versioned_docs/version-1.2.7/pipeline/intro.md",73],"080a21b4":[function(){return t.e(12).then(t.bind(null,74))},"@site/docs/components/docker_first_steps.md",74],"092c2d1d":[function(){return t.e(13).then(t.bind(null,75))},"@site/versioned_docs/version-2.1.0/pipeline/nfdump.md",75],"0b8e461a":[function(){return t.e(14).then(t.bind(null,76))},"@site/versioned_docs/version-1.2.10/pipeline/tstat.md",76],"0dd3dba6":[function(){return t.e(15).then(t.bind(null,77))},"@site/versioned_docs/version-2.1.0/devel/docker.md",77],"12ba72b5":[function(){return t.e(16).then(t.bind(null,78))},"@site/versioned_docs/version-2.1.0/deploy/docker_install_simple.md",78],"12ecfc4a":[function(){return t.e(17).then(t.bind(null,79))},"@site/versioned_docs/version-2.1.0/devel/pipeline_dataset.md",79],"16fcefd4":[function(){return t.e(18).then(t.bind(null,80))},"@site/docs/devel/tag.md",80],17896441:[function(){return Promise.all([t.e(0),t.e(1),t.e(19)]).then(t.bind(null,227))},"@theme/DocItem",227],"1be78505":[function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(157)]).then(t.bind(null,226))},"@theme/DocPage",226],"1cc43674":[function(){return t.e(20).then(t.bind(null,90))},"@site/docs/pipeline/nfdump.md",90],"1d038b8a":[function(){return t.e(21).then(t.bind(null,91))},"@site/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",91],"1f3a3353":[function(){return t.e(22).then(t.bind(null,92))},"@site/docs/devel/docker.md",92],"21ee0bc4":[function(){return t.e(23).then(t.bind(null,93))},"@site/versioned_docs/version-1.2.6/pipeline_importer.md",93],"22b7278f":[function(){return t.e(24).then(t.bind(null,94))},"@site/docs/pipeline/intro.md",94],"22d86c11":[function(){return t.e(25).then(t.bind(null,95))},"@site/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",95],"3179f6dc":[function(){return t.e(26).then(t.bind(null,96))},"@site/versioned_docs/version-1.2.5/deploy/choosing.md",96],"3309db24":[function(){return t.e(27).then(t.bind(null,97))},"@site/versioned_docs/version-1.2.7/pipeline/elastic_search.md",97],"33f6b1c9":[function(){return t.e(28).then(t.bind(null,98))},"@site/versioned_docs/version-1.2.8/pipeline/intro.md",98],"358e6759":[function(){return t.e(29).then(t.bind(null,99))},"@site/versioned_docs/version-1.2.7/components/docker_env.md",99],38962575:[function(){return t.e(30).then(t.bind(null,100))},"@site/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",100],"38d1fe39":[function(){return t.e(31).then(t.bind(null,101))},"@site/versioned_docs/version-1.2.6/pipeline_logstash.md",101],"3914abfa":[function(){return t.e(32).then(t.bind(null,102))},"@site/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",102],"40b960a4":[function(){return t.e(33).then(t.bind(null,103))},"@site/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",103],"41980e0e":[function(){return t.e(34).then(t.bind(null,104))},"@site/versioned_docs/version-1.2.11/devel/docker.md",104],"42e094bc":[function(){return t.e(35).then(t.bind(null,105))},"@site/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",105],"42f0e773":[function(){return t.e(36).then(t.bind(null,106))},"@site/versioned_docs/version-1.2.8/components/docker_pipeline.md",106],"42f7ba93":[function(){return t.e(37).then(t.bind(null,107))},"@site/versioned_docs/version-1.2.7/pipeline/nfdump.md",107],"43862f47":[function(){return t.e(38).then(t.bind(null,108))},"@site/versioned_docs/version-1.2.7/pipeline/importer.md",108],"46a600e4":[function(){return t.e(39).then(t.bind(null,109))},"@site/versioned_docs/version-1.2.10/components/docker_pipeline.md",109],"480c5911":[function(){return t.e(40).then(t.bind(null,110))},"@site/versioned_docs/version-1.2.5/DOCUMENTATION.md",110],49703371:[function(){return t.e(41).then(t.bind(null,111))},"@site/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",111],"49e74cdd":[function(){return t.e(42).then(t.bind(null,112))},"@site/versioned_docs/version-1.2.10/pipeline/intro.md",112],"4a74d8e8":[function(){return t.e(43).then(t.bind(null,113))},"@site/versioned_docs/version-1.2.10/pipeline/elastic_search.md",113],"4ff24bf8":[function(){return t.e(44).then(t.bind(null,114))},"@site/versioned_docs/version-1.2.7/components/docker_upgrade.md",114],"4ffdf56a":[function(){return t.e(45).then(t.bind(null,115))},"@site/docs/devel/documentation_guide.md",115],50032990:[function(){return t.e(46).then(t.bind(null,116))},"@site/versioned_docs/version-1.2.9/pipeline/tstat.md",116],"514d766a":[function(){return t.e(47).then(t.bind(null,117))},"@site/versioned_docs/version-1.2.10/devel/documentation_guide.md",117],"56776a50":[function(){return t.e(48).then(t.bind(null,118))},"@site/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",118],"56a29d1a":[function(){return t.e(49).then(t.bind(null,119))},"@site/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",119],"58d958f4":[function(){return t.e(50).then(t.bind(null,120))},"@site/versioned_docs/version-1.2.8/documentation_guide.md",120],"5a20f281":[function(){return t.e(51).then(t.bind(null,121))},"@site/versioned_docs/version-1.2.9/pipeline/elastic_search.md",121],"5b6c4cb7":[function(){return t.e(52).then(t.bind(null,122))},"@site/versioned_docs/version-1.2.11/pipeline/elastic_search.md",122],"5c4beb71":[function(){return t.e(53).then(t.bind(null,123))},"@site/versioned_docs/version-1.2.7/deploy/choosing.md",123],"5e9f5e1a":[function(){return Promise.resolve().then(t.bind(null,11))},"@generated/docusaurus.config",11],"5ee17acf":[function(){return t.e(54).then(t.bind(null,124))},"@site/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",124],"5ef24f1d":[function(){return t.e(55).then(t.bind(null,125))},"@site/versioned_docs/version-2.1.0/pipeline/importer.md",125],"6125e2e5":[function(){return t.e(56).then(t.bind(null,126))},"@site/versioned_docs/version-1.2.9/components/docker_env.md",126],63503174:[function(){return t.e(57).then(t.t.bind(null,127,3))},"~docs/default/version-2-1-0-metadata-prop-24f.json",127],"63f5fe43":[function(){return t.e(58).then(t.bind(null,128))},"@site/versioned_docs/version-1.2.10/devel/docker.md",128],"64269ed8":[function(){return t.e(59).then(t.bind(null,129))},"@site/docs/components/docker_env.md",129],"653602f1":[function(){return t.e(60).then(t.bind(null,130))},"@site/docs/deploy/docker_troubleshooting.md",130],"6546ef57":[function(){return t.e(61).then(t.bind(null,131))},"@site/versioned_docs/version-1.2.9/components/docker_upgrade.md",131],"69c1008e":[function(){return t.e(62).then(t.bind(null,132))},"@site/versioned_docs/version-1.2.11/components/docker_pipeline.md",132],"6a059f5b":[function(){return t.e(63).then(t.bind(null,133))},"@site/versioned_docs/version-1.2.8/pipeline/logstash.md",133],"6c279c12":[function(){return t.e(64).then(t.t.bind(null,134,3))},"~docs/default/version-1-2-5-metadata-prop-ebb.json",134],"6d913b5d":[function(){return t.e(65).then(t.bind(null,135))},"@site/versioned_docs/version-1.2.11/devel/tag.md",135],"6e2a1c72":[function(){return t.e(66).then(t.bind(null,136))},"@site/versioned_docs/version-2.1.0/components/docker_pipeline.md",136],"6e3166bc":[function(){return t.e(67).then(t.bind(null,137))},"@site/versioned_docs/version-2.1.0/devel/tag.md",137],"71e5fd50":[function(){return t.e(68).then(t.bind(null,138))},"@site/versioned_docs/version-2.1.0/pipeline/elastic_search.md",138],"734dc415":[function(){return t.e(69).then(t.bind(null,139))},"@site/docs/deploy/docker_install_simple.md",139],"799b1578":[function(){return t.e(70).then(t.bind(null,140))},"@site/versioned_docs/version-1.2.10/pipeline/importer.md",140],"7b85b03c":[function(){return t.e(71).then(t.bind(null,141))},"@site/versioned_docs/version-1.2.9/pipeline/nfdump.md",141],"7c78db09":[function(){return t.e(72).then(t.bind(null,142))},"@site/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",142],"7d0b7755":[function(){return t.e(73).then(t.bind(null,143))},"@site/docs/pipeline/elastic_search.md",143],"7d380c4c":[function(){return t.e(74).then(t.t.bind(null,144,3))},"~docs/default/version-1-2-6-metadata-prop-dbf.json",144],"832ca04e":[function(){return t.e(75).then(t.bind(null,145))},"@site/versioned_docs/version-1.2.8/pipeline/tstat.md",145],"83dbc9a5":[function(){return t.e(76).then(t.bind(null,146))},"@site/versioned_docs/version-1.2.5/devel/docker.md",146],"86d09e6e":[function(){return t.e(77).then(t.bind(null,147))},"@site/versioned_docs/version-2.1.0/deploy/docker_troubleshooting.md",147],"879923b2":[function(){return t.e(78).then(t.bind(null,148))},"@site/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",148],"8997819f":[function(){return t.e(79).then(t.bind(null,149))},"@site/versioned_docs/version-1.2.11/components/docker_env.md",149],"8b301107":[function(){return t.e(80).then(t.bind(null,150))},"@site/versioned_docs/version-2.1.0/components/docker_env.md",150],"8bbfcfa9":[function(){return t.e(81).then(t.bind(null,151))},"@site/versioned_docs/version-1.2.10/deploy/choosing.md",151],"906738b8":[function(){return t.e(82).then(t.t.bind(null,152,3))},"~docs/default/version-1-2-7-metadata-prop-988.json",152],"9191c864":[function(){return t.e(83).then(t.bind(null,153))},"@site/versioned_docs/version-2.1.0/deploy/docker_upgrade.md",153],"935f2afb":[function(){return t.e(84).then(t.t.bind(null,154,3))},"~docs/default/version-current-metadata-prop-751.json",154],96852331:[function(){return t.e(85).then(t.bind(null,155))},"@site/versioned_docs/version-1.2.11/pipeline/importer.md",155],"9720d299":[function(){return t.e(86).then(t.bind(null,156))},"@site/versioned_docs/version-1.2.9/pipeline/logstash.md",156],"97edd56e":[function(){return t.e(87).then(t.bind(null,157))},"@site/versioned_docs/version-1.2.7/devel/docker.md",157],"992819d4":[function(){return t.e(88).then(t.t.bind(null,158,3))},"~docs/default/version-1-2-8-metadata-prop-d68.json",158],"99d6017e":[function(){return t.e(89).then(t.bind(null,159))},"@site/versioned_docs/version-1.2.9/pipeline/importer.md",159],"9a5c9747":[function(){return t.e(90).then(t.bind(null,160))},"@site/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",160],"9b505a1f":[function(){return t.e(91).then(t.bind(null,161))},"@site/versioned_docs/version-1.2.9/devel/docker.md",161],"9bfdba8c":[function(){return t.e(92).then(t.bind(null,162))},"@site/versioned_docs/version-1.2.8/components/docker_first_steps.md",162],"9d5eac8e":[function(){return t.e(93).then(t.bind(null,163))},"@site/versioned_docs/version-1.2.11/devel/documentation_guide.md",163],"9f628d52":[function(){return t.e(94).then(t.bind(null,164))},"@site/versioned_docs/version-2.1.0/deploy/docker_install_advanced.md",164],"9fe55709":[function(){return t.e(95).then(t.bind(null,165))},"@site/versioned_docs/version-1.2.5/deploy/install.md",165],"9feaf8ab":[function(){return t.e(96).then(t.bind(null,166))},"@site/versioned_docs/version-1.2.6/deploy/docker_install.md",166],a04bc459:[function(){return t.e(97).then(t.bind(null,167))},"@site/versioned_docs/version-1.2.5/deploy/docker_install.md",167],a12215ea:[function(){return t.e(98).then(t.bind(null,168))},"@site/versioned_docs/version-1.2.11/pipeline/intro.md",168],a55d7fc8:[function(){return t.e(99).then(t.bind(null,169))},"@site/versioned_docs/version-1.2.8/components/docker_upgrade.md",169],a7459da2:[function(){return t.e(100).then(t.bind(null,170))},"@site/versioned_docs/version-1.2.5/pipeline_logstash.md",170],aa98ec4a:[function(){return t.e(101).then(t.bind(null,171))},"@site/versioned_docs/version-1.2.6/devel/docker.md",171],aafd28b2:[function(){return t.e(102).then(t.bind(null,172))},"@site/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",172],abbf3a21:[function(){return t.e(103).then(t.bind(null,173))},"@site/versioned_docs/version-1.2.8/components/docker_env.md",173],ac3b7d69:[function(){return t.e(104).then(t.bind(null,174))},"@site/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",174],ad308aff:[function(){return t.e(105).then(t.bind(null,175))},"@site/versioned_docs/version-1.2.8/pipeline/importer.md",175],ae8a1364:[function(){return t.e(106).then(t.bind(null,176))},"@site/versioned_docs/version-1.2.10/components/docker_first_steps.md",176],b1bfe9f5:[function(){return t.e(107).then(t.bind(null,177))},"@site/versioned_docs/version-1.2.8/pipeline/nfdump.md",177],b3c114f4:[function(){return t.e(108).then(t.bind(null,178))},"@site/versioned_docs/version-1.2.9/components/docker_first_steps.md",178],b83a1049:[function(){return t.e(109).then(t.bind(null,179))},"@site/versioned_docs/version-1.2.10/devel/tag.md",179],ba55e963:[function(){return t.e(110).then(t.bind(null,180))},"@site/docs/components/docker_pipeline.md",180],bb237b1e:[function(){return t.e(111).then(t.bind(null,181))},"@site/versioned_docs/version-1.2.7/pipeline/logstash.md",181],bd6c6f88:[function(){return t.e(112).then(t.bind(null,182))},"@site/docs/pipeline/logstash.md",182],bed7ab4e:[function(){return t.e(113).then(t.bind(null,183))},"@site/versioned_docs/version-1.2.8/pipeline/elastic_search.md",183],c0bd86e8:[function(){return t.e(114).then(t.bind(null,184))},"@site/versioned_docs/version-1.2.5/pipeline.md",184],c3f49368:[function(){return t.e(115).then(t.bind(null,185))},"@site/versioned_docs/version-1.2.9/pipeline/intro.md",185],c4f5d8e4:[function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(116)]).then(t.bind(null,186))},"@site/src/pages/index.js",186],c51f4c72:[function(){return t.e(117).then(t.bind(null,188))},"@site/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",188],cbd1c0d0:[function(){return t.e(118).then(t.bind(null,189))},"@site/docs/deploy/docker_install_advanced.md",189],cd902920:[function(){return t.e(119).then(t.bind(null,190))},"@site/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",190],ceb45c47:[function(){return t.e(120).then(t.bind(null,191))},"@site/docs/deploy/docker_upgrade.md",191],d1597237:[function(){return t.e(121).then(t.bind(null,192))},"@site/versioned_docs/version-1.2.9/components/docker_pipeline.md",192],d2c4f39e:[function(){return t.e(122).then(t.bind(null,193))},"@site/versioned_docs/version-1.2.8/deploy/choosing.md",193],d3556f45:[function(){return t.e(123).then(t.bind(null,194))},"@site/docs/pipeline/importer.md",194],d48b255e:[function(){return t.e(124).then(t.bind(null,195))},"@site/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",195],d5087a67:[function(){return t.e(125).then(t.t.bind(null,196,3))},"~docs/default/version-1-2-10-metadata-prop-e71.json",196],d5140579:[function(){return t.e(126).then(t.bind(null,197))},"@site/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",197],d5479f42:[function(){return t.e(127).then(t.bind(null,198))},"@site/versioned_docs/version-1.2.6/deploy/choosing.md",198],d71ff009:[function(){return t.e(128).then(t.bind(null,199))},"@site/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",199],d74c151a:[function(){return t.e(129).then(t.bind(null,200))},"@site/versioned_docs/version-2.1.0/devel/documentation_guide.md",200],d75898ab:[function(){return t.e(130).then(t.t.bind(null,201,3))},"~docs/default/version-1-2-9-metadata-prop-33d.json",201],d91f9def:[function(){return t.e(131).then(t.bind(null,202))},"@site/versioned_docs/version-1.2.6/DOCUMENTATION.md",202],db0ed613:[function(){return t.e(132).then(t.bind(null,203))},"@site/versioned_docs/version-1.2.9/documentation_guide.md",203],dd1a9f26:[function(){return t.e(133).then(t.bind(null,204))},"@site/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",204],e00e17a0:[function(){return t.e(134).then(t.bind(null,205))},"@site/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",205],e10970d8:[function(){return t.e(135).then(t.bind(null,206))},"@site/versioned_docs/version-1.2.10/components/docker_env.md",206],e186625e:[function(){return t.e(136).then(t.bind(null,207))},"@site/versioned_docs/version-1.2.7/pipeline/tstat.md",207],e2be698d:[function(){return t.e(137).then(t.bind(null,208))},"@site/versioned_docs/version-1.2.6/deploy/install.md",208],e2c161d9:[function(){return t.e(138).then(t.bind(null,209))},"@site/docs/pipeline/tstat.md",209],e30bbbcf:[function(){return t.e(139).then(t.bind(null,210))},"@site/versioned_docs/version-1.2.11/pipeline/nfdump.md",210],e436ce66:[function(){return t.e(140).then(t.bind(null,211))},"@site/versioned_docs/version-1.2.7/components/docker_pipeline.md",211],e746408e:[function(){return t.e(141).then(t.bind(null,212))},"@site/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",212],e7670743:[function(){return t.e(142).then(t.bind(null,213))},"@site/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",213],e987ec87:[function(){return t.e(143).then(t.bind(null,214))},"@site/versioned_docs/version-1.2.11/components/docker_first_steps.md",214],ebfaece6:[function(){return t.e(144).then(t.bind(null,215))},"@site/versioned_docs/version-1.2.6/pipeline.md",215],f0b0282f:[function(){return t.e(145).then(t.bind(null,216))},"@site/versioned_docs/version-2.1.0/pipeline/logstash.md",216],f1ddb0ed:[function(){return t.e(146).then(t.bind(null,217))},"@site/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",217],f2cce206:[function(){return t.e(147).then(t.bind(null,218))},"@site/versioned_docs/version-2.1.0/pipeline/intro.md",218],f485eca7:[function(){return t.e(148).then(t.bind(null,219))},"@site/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",219],f52ba0ac:[function(){return t.e(149).then(t.bind(null,220))},"@site/versioned_docs/version-1.2.11/pipeline/tstat.md",220],f64b3473:[function(){return t.e(150).then(t.bind(null,221))},"@site/versioned_docs/version-1.2.9/deploy/choosing.md",221],f8bc0717:[function(){return t.e(151).then(t.bind(null,222))},"@site/versioned_docs/version-1.2.10/pipeline/nfdump.md",222],f9995f72:[function(){return t.e(152).then(t.bind(null,223))},"@site/versioned_docs/version-2.1.0/pipeline/tstat.md",223],fa16f0fc:[function(){return t.e(153).then(t.bind(null,224))},"@site/docs/devel/pipeline_dataset.md",224],ff1d9e71:[function(){return t.e(154).then(t.bind(null,225))},"@site/versioned_docs/version-2.1.0/components/docker_first_steps.md",225]};var d=function(e){var n={};return function e(t,i){Object.keys(t).forEach((function(r){var o=t[r],a=i?i+"."+r:r;"object"===typeof o&&!!o&&Object.keys(o).length?e(o,a):n[a]=o}))}(e),n};var f=function(e,n){if("*"===e)return s()({loading:c,loader:function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(158)]).then(t.bind(null,239))}});var i=u[e+"-"+n],o=[],a=[],l={},f=d(i);return Object.keys(f).forEach((function(e){var n=p[f[e]];n&&(l[e]=n[0],o.push(n[1]),a.push(n[2]))})),s.a.Map({loading:c,loader:l,modules:o,webpack:function(){return a},render:function(e,n){var t=JSON.parse(JSON.stringify(i));Object.keys(e).forEach((function(n){for(var i=t,r=n.split("."),o=0;o<r.length-1;o+=1)i=i[r[o]];i[r[r.length-1]]=e[n].default;var a=Object.keys(e[n]).filter((function(e){return"default"!==e}));a&&a.length&&a.forEach((function(t){i[r[r.length-1]][t]=e[n][t]}))}));var o=t.component;return delete t.component,r.a.createElement(o,Object.assign({},t,n))}})},m=[{path:"/netsage-pipeline/",component:f("/netsage-pipeline/","421"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10",component:f("/netsage-pipeline/docs/1.2.10","296"),routes:[{path:"/netsage-pipeline/docs/1.2.10/components/docker_env",component:f("/netsage-pipeline/docs/1.2.10/components/docker_env","63f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.10/components/docker_first_steps","059"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.10/components/docker_pipeline","a0b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install","ec8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.10/deploy/choose_install","fb2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced","a83"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple","15f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot","e2e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade","b01"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.10/devel/dev_dataset","7fa"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide","f40"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag","b8f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docusaurus",component:f("/netsage-pipeline/docs/1.2.10/devel/docusaurus","bcf"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.10/pipeline/elastic","c07"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.10/pipeline/importer","3c0"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.10/pipeline/intro","3b3"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.10/pipeline/logstash","116"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.10/pipeline/nfdump","f4d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.10/pipeline/tstat","4c2"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.11",component:f("/netsage-pipeline/docs/1.2.11","d04"),routes:[{path:"/netsage-pipeline/docs/1.2.11/components/docker_env",component:f("/netsage-pipeline/docs/1.2.11/components/docker_env","1d6"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.11/components/docker_first_steps","62e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.11/components/docker_pipeline","6b1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install","c46"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.11/deploy/choose_install","f72"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced","0a8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple","c8a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot","a22"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade","9ae"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.11/devel/dev_dataset","ba4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide","80d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag","681"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docusaurus",component:f("/netsage-pipeline/docs/1.2.11/devel/docusaurus","e33"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.11/pipeline/elastic","ef5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.11/pipeline/importer","5cc"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.11/pipeline/intro","5fd"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.11/pipeline/logstash","0d3"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.11/pipeline/nfdump","9b4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.11/pipeline/tstat","3cb"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.5",component:f("/netsage-pipeline/docs/1.2.5","c52"),routes:[{path:"/netsage-pipeline/docs/1.2.5/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.5/deploy/choose_install","3ce"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/deploy/docker_install",component:f("/netsage-pipeline/docs/1.2.5/deploy/docker_install","d98"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/deploy/install",component:f("/netsage-pipeline/docs/1.2.5/deploy/install","47c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/devel/docker",component:f("/netsage-pipeline/docs/1.2.5/devel/docker","0e4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/docusaurus",component:f("/netsage-pipeline/docs/1.2.5/docusaurus","882"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline",component:f("/netsage-pipeline/docs/1.2.5/pipeline","8ff"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline_importer",component:f("/netsage-pipeline/docs/1.2.5/pipeline_importer","93b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline_logstash",component:f("/netsage-pipeline/docs/1.2.5/pipeline_logstash","9ed"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.6",component:f("/netsage-pipeline/docs/1.2.6","489"),routes:[{path:"/netsage-pipeline/docs/1.2.6/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.6/deploy/choose_install","0d1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/deploy/docker_install",component:f("/netsage-pipeline/docs/1.2.6/deploy/docker_install","4df"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/deploy/install",component:f("/netsage-pipeline/docs/1.2.6/deploy/install","8ee"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/devel/docker",component:f("/netsage-pipeline/docs/1.2.6/devel/docker","d95"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/docusaurus",component:f("/netsage-pipeline/docs/1.2.6/docusaurus","312"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline",component:f("/netsage-pipeline/docs/1.2.6/pipeline","7ea"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline_importer",component:f("/netsage-pipeline/docs/1.2.6/pipeline_importer","c76"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline_logstash",component:f("/netsage-pipeline/docs/1.2.6/pipeline_logstash","ee6"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.7",component:f("/netsage-pipeline/docs/1.2.7","3e0"),routes:[{path:"/netsage-pipeline/docs/1.2.7/components/docker_env",component:f("/netsage-pipeline/docs/1.2.7/components/docker_env","fb9"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.7/components/docker_pipeline","28e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.7/components/docker_upgrade","aca"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install","88b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.7/deploy/choose_install","9c2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced","a68"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple","c15"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot","f5e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.7/devel/dev_dataset","c2f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide","13c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/docusaurus",component:f("/netsage-pipeline/docs/1.2.7/docusaurus","fba"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.7/pipeline/elastic","545"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.7/pipeline/importer","457"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.7/pipeline/intro","d5a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.7/pipeline/logstash","144"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.7/pipeline/nfdump","6d8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.7/pipeline/tstat","d6c"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.8",component:f("/netsage-pipeline/docs/1.2.8","51d"),routes:[{path:"/netsage-pipeline/docs/1.2.8/components/docker_env",component:f("/netsage-pipeline/docs/1.2.8/components/docker_env","f3f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.8/components/docker_first_steps","d25"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.8/components/docker_pipeline","7d7"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.8/components/docker_upgrade","db5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install","a0d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.8/deploy/choose_install","298"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced","150"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple","f57"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot","a18"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.8/devel/dev_dataset","744"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide","e12"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/docusaurus",component:f("/netsage-pipeline/docs/1.2.8/docusaurus","57f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.8/pipeline/elastic","e1a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.8/pipeline/importer","b38"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.8/pipeline/intro","c0b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.8/pipeline/logstash","00d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.8/pipeline/nfdump","6f1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.8/pipeline/tstat","59a"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.9",component:f("/netsage-pipeline/docs/1.2.9","cb5"),routes:[{path:"/netsage-pipeline/docs/1.2.9/components/docker_env",component:f("/netsage-pipeline/docs/1.2.9/components/docker_env","e31"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.9/components/docker_first_steps","253"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.9/components/docker_pipeline","6e2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.9/components/docker_upgrade","8be"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install","31c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.9/deploy/choose_install","5ff"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced","711"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple","536"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot","7d0"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.9/devel/dev_dataset","e4a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide","b39"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/docusaurus",component:f("/netsage-pipeline/docs/1.2.9/docusaurus","bed"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.9/pipeline/elastic","bd5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.9/pipeline/importer","90f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.9/pipeline/intro","e91"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.9/pipeline/logstash","9e9"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.9/pipeline/nfdump","220"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.9/pipeline/tstat","341"),exact:!0}]},{path:"/netsage-pipeline/docs/next",component:f("/netsage-pipeline/docs/next","47f"),routes:[{path:"/netsage-pipeline/docs/next/components/docker_env",component:f("/netsage-pipeline/docs/next/components/docker_env","cfe"),exact:!0},{path:"/netsage-pipeline/docs/next/components/docker_first_steps",component:f("/netsage-pipeline/docs/next/components/docker_first_steps","e72"),exact:!0},{path:"/netsage-pipeline/docs/next/components/docker_pipeline",component:f("/netsage-pipeline/docs/next/components/docker_pipeline","5eb"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/next/deploy/docker_install_advanced","de0"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/next/deploy/docker_install_simple","72d"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/next/deploy/docker_troubleshoot","593"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/next/deploy/docker_upgrade","68f"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/dev_dataset",component:f("/netsage-pipeline/docs/next/devel/dev_dataset","091"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/next/devel/docker_dev_guide","85f"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/next/devel/docker_dev_tag","e93"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docusaurus",component:f("/netsage-pipeline/docs/next/devel/docusaurus","702"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/elastic",component:f("/netsage-pipeline/docs/next/pipeline/elastic","533"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/importer",component:f("/netsage-pipeline/docs/next/pipeline/importer","063"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/intro",component:f("/netsage-pipeline/docs/next/pipeline/intro","f04"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/logstash",component:f("/netsage-pipeline/docs/next/pipeline/logstash","aec"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/nfdump",component:f("/netsage-pipeline/docs/next/pipeline/nfdump","543"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/tstat",component:f("/netsage-pipeline/docs/next/pipeline/tstat","474"),exact:!0}]},{path:"/netsage-pipeline/docs",component:f("/netsage-pipeline/docs","0a0"),routes:[{path:"/netsage-pipeline/docs/components/docker_env",component:f("/netsage-pipeline/docs/components/docker_env","a6d"),exact:!0},{path:"/netsage-pipeline/docs/components/docker_first_steps",component:f("/netsage-pipeline/docs/components/docker_first_steps","fc9"),exact:!0},{path:"/netsage-pipeline/docs/components/docker_pipeline",component:f("/netsage-pipeline/docs/components/docker_pipeline","40b"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/deploy/docker_install_advanced","de0"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/deploy/docker_install_simple","6e9"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/deploy/docker_troubleshoot","8df"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/deploy/docker_upgrade","bcf"),exact:!0},{path:"/netsage-pipeline/docs/devel/dev_dataset",component:f("/netsage-pipeline/docs/devel/dev_dataset","e5f"),exact:!0},{path:"/netsage-pipeline/docs/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/devel/docker_dev_guide","81e"),exact:!0},{path:"/netsage-pipeline/docs/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/devel/docker_dev_tag","8f8"),exact:!0},{path:"/netsage-pipeline/docs/devel/docusaurus",component:f("/netsage-pipeline/docs/devel/docusaurus","78c"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/elastic",component:f("/netsage-pipeline/docs/pipeline/elastic","d63"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/importer",component:f("/netsage-pipeline/docs/pipeline/importer","e01"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/intro",component:f("/netsage-pipeline/docs/pipeline/intro","85d"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/logstash",component:f("/netsage-pipeline/docs/pipeline/logstash","bf4"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/nfdump",component:f("/netsage-pipeline/docs/pipeline/nfdump","adf"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/tstat",component:f("/netsage-pipeline/docs/pipeline/tstat","9b0"),exact:!0}]},{path:"*",component:f("*")}],g=t(8),h=t(11),v=t(31),b=t(32),y=t(26),k=t(33),w=t(27),_=t(14),x=t(4),E=t(2),S=t(15),T=t.n(S),P=[t(18),t(18),t(18),t(47),t(55),t(51)];function C(e){for(var n=arguments.length,t=new Array(n>1?n-1:0),i=1;i<n;i++)t[i-1]=arguments[i];P.forEach((function(n){var i,r,o=null!==(r=null===(i=null==n?void 0:n.default)||void 0===i?void 0:i[e])&&void 0!==r?r:n[e];o&&o.apply(void 0,t)}))}var O={onRouteUpdate:function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];C.apply(void 0,["onRouteUpdate"].concat(n))},onRouteUpdateDelayed:function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];C.apply(void 0,["onRouteUpdateDelayed"].concat(n))}},A=t(9);function R(e,n){var t=Object(A.a)(e,n);return Promise.all(t.map((function(e){var n=e.route.component;if(n&&n.preload)return n.preload()})))}var N={};var I=function(e){if(N[e.pathname])return Object.assign(Object.assign({},e),{pathname:N[e.pathname]});var n=e.pathname||"/";return""===(n=n.trim().replace(/\/index\.html$/,""))&&(n="/"),N[e.pathname]=n,Object.assign(Object.assign({},e),{pathname:n})};t(52);T.a.configure({showSpinner:!1});var L=function(e){function n(n){var t;return(t=e.call(this,n)||this).previousLocation=null,t.progressBarTimeout=null,t.state={nextRouteHasLoaded:!0},t}Object(x.a)(n,e);var t=n.prototype;return t.shouldComponentUpdate=function(e,n){var t=this,i=e.location!==this.props.location,r=this.props,o=r.routes,a=r.delay,l=void 0===a?1e3:a;if(i){var s=I(e.location);return this.startProgressBar(l),this.previousLocation=I(this.props.location),this.setState({nextRouteHasLoaded:!1}),R(o,s.pathname).then((function(){O.onRouteUpdate({previousLocation:t.previousLocation,location:s}),t.previousLocation=null,t.setState({nextRouteHasLoaded:!0},t.stopProgressBar);var e=s.hash;if(e){var n=decodeURIComponent(e.substring(1)),i=document.getElementById(n);i&&i.scrollIntoView()}else window.scrollTo(0,0)})).catch((function(e){return console.warn(e)})),!1}return!!n.nextRouteHasLoaded},t.clearProgressBarTimeout=function(){this.progressBarTimeout&&(clearTimeout(this.progressBarTimeout),this.progressBarTimeout=null)},t.startProgressBar=function(e){var n=this;this.clearProgressBarTimeout(),this.progressBarTimeout=setTimeout((function(){O.onRouteUpdateDelayed({location:I(n.props.location)}),T.a.start()}),e)},t.stopProgressBar=function(){this.clearProgressBarTimeout(),T.a.done()},t.render=function(){var e=this.props,n=e.children,t=e.location;return r.a.createElement(E.d,{location:I(t),render:function(){return n}})},n}(r.a.Component),D=Object(E.o)(L),j=t(23),M=t(25),F=t(16),B=(t(53),"docusaurus-base-url-issue-banner-container"),z="docusaurus-base-url-issue-banner-suggestion-container",U="__DOCUSAURUS_INSERT_BASEURL_BANNER";function $(e){return"\nwindow['"+U+"'] = true;\n\ndocument.addEventListener('DOMContentLoaded', maybeInsertBanner);\n\nfunction maybeInsertBanner() {\n  var shouldInsert = window['"+U+"'];\n  shouldInsert && insertBanner();\n}\n\nfunction insertBanner() {\n  var bannerContainer = document.getElementById('"+B+"');\n  if (!bannerContainer) {\n    return;\n  }\n  var bannerHtml = "+JSON.stringify(function(e){return'\n<div id="docusaurus-base-url-issue-banner" style="border: thick solid red; background-color: rgb(255, 230, 179); margin: 20px; padding: 20px; font-size: 20px;">\n   <p style="font-weight: bold; font-size: 30px;">Your Docusaurus site did not load properly.</p>\n   <p>A very common reason is a wrong site <a href="https://v2.docusaurus.io/docs/docusaurus.config.js/#baseurl" style="font-weight: bold;">baseUrl configuration</a>.</p>\n   <p>Current configured baseUrl = <span style="font-weight: bold; color: red;">'+e+"</span> "+("/"===e?" (default value)":"")+'</p>\n   <p>We suggest trying baseUrl = <span id="'+z+'" style="font-weight: bold; color: green;"></span></p>\n</div>\n'}(e)).replace(/</g,"\\<")+";\n  bannerContainer.innerHTML = bannerHtml;\n  var suggestionContainer = document.getElementById('"+z+"');\n  var actualHomePagePath = window.location.pathname;\n  var suggestedBaseUrl = actualHomePagePath.substr(-1) === '/'\n        ? actualHomePagePath\n        : actualHomePagePath + '/';\n  suggestionContainer.innerHTML = suggestedBaseUrl;\n}\n"}function G(){var e=Object(F.default)().siteConfig.baseUrl;return Object(i.useLayoutEffect)((function(){window[U]=!1}),[]),r.a.createElement(r.a.Fragment,null,!g.a.canUseDOM&&r.a.createElement(M.a,null,r.a.createElement("script",null,$(e))),r.a.createElement("div",{id:B}))}function q(){var e=Object(F.default)().siteConfig,n=e.baseUrl,t=e.baseUrlIssueBanner,i=Object(j.useLocation)().pathname;return t&&i===n?r.a.createElement(G,null):null}var H=function(e){var n=e.children;return r.a.createElement(r.a.Fragment,null,n)};var W=function(){var e=Object(i.useState)(!1),n=e[0],t=e[1];return Object(i.useEffect)((function(){t(!0)}),[]),r.a.createElement(_.a.Provider,{value:{siteConfig:h.default,siteMetadata:k,globalData:v,i18n:b,codeTranslations:y,isClient:n}},r.a.createElement(H,null,r.a.createElement(q,null),r.a.createElement(D,{routes:m},Object(w.a)(m))))};var V=function(e){if("undefined"==typeof document)return!1;var n=document.createElement("link");try{if(n.relList&&"function"==typeof n.relList.supports)return n.relList.supports(e)}catch(t){return!1}return!1}("prefetch")?function(e){return new Promise((function(n,t){if("undefined"!=typeof document){var i=document.createElement("link");i.setAttribute("rel","prefetch"),i.setAttribute("href",e),i.onload=n,i.onerror=t,(document.getElementsByTagName("head")[0]||document.getElementsByName("script")[0].parentNode).appendChild(i)}else t()}))}:function(e){return new Promise((function(n,t){var i=new XMLHttpRequest;i.open("GET",e,!0),i.withCredentials=!0,i.onload=function(){200===i.status?n():t()},i.send(null)}))},K={};var Y=function(e){return new Promise((function(n){K[e]?n():V(e).then((function(){n(),K[e]=!0})).catch((function(){}))}))},Q={},X={},J=function(){return!(!("connection"in navigator)||-1===(navigator.connection.effectiveType||"").indexOf("2g")||!navigator.connection.saveData)},Z=function(e){return Array.prototype.concat.apply([],e)},ee={prefetch:function(e){if(!function(e){return!J()&&!X[e]&&!Q[e]}(e))return!1;Q[e]=!0;var n=Object(A.a)(m,e);return Z(n.map((function(e){return n=e.route.path,Z(Object.entries(u).filter((function(e){return e[0].replace(/(-[^-]+)$/,"")===n})).map((function(e){var n=e[1];return Object.values(d(n))})));var n}))).forEach((function(e){var n=t.gca(e);n&&!/undefined/.test(n)&&Y(n)})),!0},preload:function(e){return!!function(e){return!J()&&!X[e]}(e)&&(X[e]=!0,R(m,e),!0)}};if(g.a.canUseDOM){window.docusaurus=ee;var ne=o.hydrate;R(m,window.location.pathname).then((function(){ne(r.a.createElement(a.a,null,r.a.createElement(W,null)),document.getElementById("__docusaurus"))}))}},function(e,n,t){"use strict";t.r(n);var i=t(24),r=t(8),o=t(11);(function(e){if(r.a.canUseDOM){var n=o.default.themeConfig.prism,i=(n=void 0===n?{}:n).additionalLanguages,a=void 0===i?[]:i;window.Prism=e,a.forEach((function(e){t(19)("./prism-"+e)})),delete window.Prism}})(i.a)}],[[38,156,0]]]);
\ No newline at end of file
diff --git a/assets/js/main.e20b6a1c.js.LICENSE.txt b/assets/js/main.16ba1fad.js.LICENSE.txt
similarity index 100%
rename from assets/js/main.e20b6a1c.js.LICENSE.txt
rename to assets/js/main.16ba1fad.js.LICENSE.txt
diff --git a/assets/js/main.e20b6a1c.js b/assets/js/main.e20b6a1c.js
deleted file mode 100644
index 28d16e8e..00000000
--- a/assets/js/main.e20b6a1c.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! For license information please see main.e20b6a1c.js.LICENSE.txt */
-(window.webpackJsonp=window.webpackJsonp||[]).push([[159],[function(e,n,t){"use strict";e.exports=t(39)},function(e,n,t){e.exports=t(43)()},function(e,n,t){"use strict";t.d(n,"a",(function(){return b})),t.d(n,"b",(function(){return k})),t.d(n,"c",(function(){return E})),t.d(n,"d",(function(){return C})),t.d(n,"e",(function(){return v})),t.d(n,"f",(function(){return L})),t.d(n,"g",(function(){return D})),t.d(n,"h",(function(){return h})),t.d(n,"i",(function(){return x})),t.d(n,"j",(function(){return P})),t.d(n,"k",(function(){return F})),t.d(n,"l",(function(){return B})),t.d(n,"m",(function(){return z})),t.d(n,"n",(function(){return U})),t.d(n,"o",(function(){return j}));var i=t(4),r=t(0),o=t.n(r),a=t(6),l=t(21),s=t(5),c=t(3),u=t(22),p=t.n(u),d=(t(29),t(7)),f=t(34),m=t.n(f),g=function(e){var n=Object(l.a)();return n.displayName=e,n}("Router-History"),h=function(e){var n=Object(l.a)();return n.displayName=e,n}("Router"),v=function(e){function n(n){var t;return(t=e.call(this,n)||this).state={location:n.history.location},t._isMounted=!1,t._pendingLocation=null,n.staticContext||(t.unlisten=n.history.listen((function(e){t._isMounted?t.setState({location:e}):t._pendingLocation=e}))),t}Object(i.a)(n,e),n.computeRootMatch=function(e){return{path:"/",url:"/",params:{},isExact:"/"===e}};var t=n.prototype;return t.componentDidMount=function(){this._isMounted=!0,this._pendingLocation&&this.setState({location:this._pendingLocation})},t.componentWillUnmount=function(){this.unlisten&&this.unlisten()},t.render=function(){return o.a.createElement(h.Provider,{value:{history:this.props.history,location:this.state.location,match:n.computeRootMatch(this.state.location.pathname),staticContext:this.props.staticContext}},o.a.createElement(g.Provider,{children:this.props.children||null,value:this.props.history}))},n}(o.a.Component);var b=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(a.d)(n.props),n}return Object(i.a)(n,e),n.prototype.render=function(){return o.a.createElement(v,{history:this.history,children:this.props.children})},n}(o.a.Component);var y=function(e){function n(){return e.apply(this,arguments)||this}Object(i.a)(n,e);var t=n.prototype;return t.componentDidMount=function(){this.props.onMount&&this.props.onMount.call(this,this)},t.componentDidUpdate=function(e){this.props.onUpdate&&this.props.onUpdate.call(this,this,e)},t.componentWillUnmount=function(){this.props.onUnmount&&this.props.onUnmount.call(this,this)},t.render=function(){return null},n}(o.a.Component);function k(e){var n=e.message,t=e.when,i=void 0===t||t;return o.a.createElement(h.Consumer,null,(function(e){if(e||Object(s.a)(!1),!i||e.staticContext)return null;var t=e.history.block;return o.a.createElement(y,{onMount:function(e){e.release=t(n)},onUpdate:function(e,i){i.message!==n&&(e.release(),e.release=t(n))},onUnmount:function(e){e.release()},message:n})}))}var w={},_=0;function x(e,n){return void 0===e&&(e="/"),void 0===n&&(n={}),"/"===e?e:function(e){if(w[e])return w[e];var n=p.a.compile(e);return _<1e4&&(w[e]=n,_++),n}(e)(n,{pretty:!0})}function E(e){var n=e.computedMatch,t=e.to,i=e.push,r=void 0!==i&&i;return o.a.createElement(h.Consumer,null,(function(e){e||Object(s.a)(!1);var i=e.history,l=e.staticContext,u=r?i.push:i.replace,p=Object(a.c)(n?"string"==typeof t?x(t,n.params):Object(c.a)({},t,{pathname:x(t.pathname,n.params)}):t);return l?(u(p),null):o.a.createElement(y,{onMount:function(){u(p)},onUpdate:function(e,n){var t=Object(a.c)(n.to);Object(a.f)(t,Object(c.a)({},p,{key:t.key}))||u(p)},to:t})}))}var S={},T=0;function P(e,n){void 0===n&&(n={}),("string"==typeof n||Array.isArray(n))&&(n={path:n});var t=n,i=t.path,r=t.exact,o=void 0!==r&&r,a=t.strict,l=void 0!==a&&a,s=t.sensitive,c=void 0!==s&&s;return[].concat(i).reduce((function(n,t){if(!t&&""!==t)return null;if(n)return n;var i=function(e,n){var t=""+n.end+n.strict+n.sensitive,i=S[t]||(S[t]={});if(i[e])return i[e];var r=[],o={regexp:p()(e,r,n),keys:r};return T<1e4&&(i[e]=o,T++),o}(t,{end:o,strict:l,sensitive:c}),r=i.regexp,a=i.keys,s=r.exec(e);if(!s)return null;var u=s[0],d=s.slice(1),f=e===u;return o&&!f?null:{path:t,url:"/"===t&&""===u?"/":u,isExact:f,params:a.reduce((function(e,n,t){return e[n.name]=d[t],e}),{})}}),null)}var C=function(e){function n(){return e.apply(this,arguments)||this}return Object(i.a)(n,e),n.prototype.render=function(){var e=this;return o.a.createElement(h.Consumer,null,(function(n){n||Object(s.a)(!1);var t=e.props.location||n.location,i=e.props.computedMatch?e.props.computedMatch:e.props.path?P(t.pathname,e.props):n.match,r=Object(c.a)({},n,{location:t,match:i}),a=e.props,l=a.children,u=a.component,p=a.render;return Array.isArray(l)&&0===l.length&&(l=null),o.a.createElement(h.Provider,{value:r},r.match?l?"function"==typeof l?l(r):l:u?o.a.createElement(u,r):p?p(r):null:"function"==typeof l?l(r):null)}))},n}(o.a.Component);function O(e){return"/"===e.charAt(0)?e:"/"+e}function A(e,n){if(!e)return n;var t=O(e);return 0!==n.pathname.indexOf(t)?n:Object(c.a)({},n,{pathname:n.pathname.substr(t.length)})}function R(e){return"string"==typeof e?e:Object(a.e)(e)}function N(e){return function(){Object(s.a)(!1)}}function I(){}var L=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).handlePush=function(e){return n.navigateTo(e,"PUSH")},n.handleReplace=function(e){return n.navigateTo(e,"REPLACE")},n.handleListen=function(){return I},n.handleBlock=function(){return I},n}Object(i.a)(n,e);var t=n.prototype;return t.navigateTo=function(e,n){var t=this.props,i=t.basename,r=void 0===i?"":i,o=t.context,l=void 0===o?{}:o;l.action=n,l.location=function(e,n){return e?Object(c.a)({},n,{pathname:O(e)+n.pathname}):n}(r,Object(a.c)(e)),l.url=R(l.location)},t.render=function(){var e=this.props,n=e.basename,t=void 0===n?"":n,i=e.context,r=void 0===i?{}:i,l=e.location,s=void 0===l?"/":l,u=Object(d.a)(e,["basename","context","location"]),p={createHref:function(e){return O(t+R(e))},action:"POP",location:A(t,Object(a.c)(s)),push:this.handlePush,replace:this.handleReplace,go:N(),goBack:N(),goForward:N(),listen:this.handleListen,block:this.handleBlock};return o.a.createElement(v,Object(c.a)({},u,{history:p,staticContext:r}))},n}(o.a.Component);var D=function(e){function n(){return e.apply(this,arguments)||this}return Object(i.a)(n,e),n.prototype.render=function(){var e=this;return o.a.createElement(h.Consumer,null,(function(n){n||Object(s.a)(!1);var t,i,r=e.props.location||n.location;return o.a.Children.forEach(e.props.children,(function(e){if(null==i&&o.a.isValidElement(e)){t=e;var a=e.props.path||e.props.from;i=a?P(r.pathname,Object(c.a)({},e.props,{path:a})):n.match}})),i?o.a.cloneElement(t,{location:r,computedMatch:i}):null}))},n}(o.a.Component);function j(e){var n="withRouter("+(e.displayName||e.name)+")",t=function(n){var t=n.wrappedComponentRef,i=Object(d.a)(n,["wrappedComponentRef"]);return o.a.createElement(h.Consumer,null,(function(n){return n||Object(s.a)(!1),o.a.createElement(e,Object(c.a)({},i,n,{ref:t}))}))};return t.displayName=n,t.WrappedComponent=e,m()(t,e)}var M=o.a.useContext;function F(){return M(g)}function B(){return M(h).location}function z(){var e=M(h).match;return e?e.params:{}}function U(e){var n=B(),t=M(h).match;return e?P(n.pathname,e):t}},function(e,n,t){"use strict";function i(){return(i=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e}).apply(this,arguments)}t.d(n,"a",(function(){return i}))},function(e,n,t){"use strict";function i(e,n){return(i=Object.setPrototypeOf||function(e,n){return e.__proto__=n,e})(e,n)}function r(e,n){e.prototype=Object.create(n.prototype),e.prototype.constructor=e,i(e,n)}t.d(n,"a",(function(){return r}))},function(e,n,t){"use strict";var i="Invariant failed";n.a=function(e,n){if(!e)throw new Error(i)}},function(e,n,t){"use strict";t.d(n,"a",(function(){return x})),t.d(n,"b",(function(){return O})),t.d(n,"d",(function(){return R})),t.d(n,"c",(function(){return g})),t.d(n,"f",(function(){return h})),t.d(n,"e",(function(){return m}));var i=t(3);function r(e){return"/"===e.charAt(0)}function o(e,n){for(var t=n,i=t+1,r=e.length;i<r;t+=1,i+=1)e[t]=e[i];e.pop()}var a=function(e,n){void 0===n&&(n="");var t,i=e&&e.split("/")||[],a=n&&n.split("/")||[],l=e&&r(e),s=n&&r(n),c=l||s;if(e&&r(e)?a=i:i.length&&(a.pop(),a=a.concat(i)),!a.length)return"/";if(a.length){var u=a[a.length-1];t="."===u||".."===u||""===u}else t=!1;for(var p=0,d=a.length;d>=0;d--){var f=a[d];"."===f?o(a,d):".."===f?(o(a,d),p++):p&&(o(a,d),p--)}if(!c)for(;p--;p)a.unshift("..");!c||""===a[0]||a[0]&&r(a[0])||a.unshift("");var m=a.join("/");return t&&"/"!==m.substr(-1)&&(m+="/"),m};function l(e){return e.valueOf?e.valueOf():Object.prototype.valueOf.call(e)}var s=function e(n,t){if(n===t)return!0;if(null==n||null==t)return!1;if(Array.isArray(n))return Array.isArray(t)&&n.length===t.length&&n.every((function(n,i){return e(n,t[i])}));if("object"==typeof n||"object"==typeof t){var i=l(n),r=l(t);return i!==n||r!==t?e(i,r):Object.keys(Object.assign({},n,t)).every((function(i){return e(n[i],t[i])}))}return!1},c=t(5);function u(e){return"/"===e.charAt(0)?e:"/"+e}function p(e){return"/"===e.charAt(0)?e.substr(1):e}function d(e,n){return function(e,n){return 0===e.toLowerCase().indexOf(n.toLowerCase())&&-1!=="/?#".indexOf(e.charAt(n.length))}(e,n)?e.substr(n.length):e}function f(e){return"/"===e.charAt(e.length-1)?e.slice(0,-1):e}function m(e){var n=e.pathname,t=e.search,i=e.hash,r=n||"/";return t&&"?"!==t&&(r+="?"===t.charAt(0)?t:"?"+t),i&&"#"!==i&&(r+="#"===i.charAt(0)?i:"#"+i),r}function g(e,n,t,r){var o;"string"==typeof e?(o=function(e){var n=e||"/",t="",i="",r=n.indexOf("#");-1!==r&&(i=n.substr(r),n=n.substr(0,r));var o=n.indexOf("?");return-1!==o&&(t=n.substr(o),n=n.substr(0,o)),{pathname:n,search:"?"===t?"":t,hash:"#"===i?"":i}}(e)).state=n:(void 0===(o=Object(i.a)({},e)).pathname&&(o.pathname=""),o.search?"?"!==o.search.charAt(0)&&(o.search="?"+o.search):o.search="",o.hash?"#"!==o.hash.charAt(0)&&(o.hash="#"+o.hash):o.hash="",void 0!==n&&void 0===o.state&&(o.state=n));try{o.pathname=decodeURI(o.pathname)}catch(l){throw l instanceof URIError?new URIError('Pathname "'+o.pathname+'" could not be decoded. This is likely caused by an invalid percent-encoding.'):l}return t&&(o.key=t),r?o.pathname?"/"!==o.pathname.charAt(0)&&(o.pathname=a(o.pathname,r.pathname)):o.pathname=r.pathname:o.pathname||(o.pathname="/"),o}function h(e,n){return e.pathname===n.pathname&&e.search===n.search&&e.hash===n.hash&&e.key===n.key&&s(e.state,n.state)}function v(){var e=null;var n=[];return{setPrompt:function(n){return e=n,function(){e===n&&(e=null)}},confirmTransitionTo:function(n,t,i,r){if(null!=e){var o="function"==typeof e?e(n,t):e;"string"==typeof o?"function"==typeof i?i(o,r):r(!0):r(!1!==o)}else r(!0)},appendListener:function(e){var t=!0;function i(){t&&e.apply(void 0,arguments)}return n.push(i),function(){t=!1,n=n.filter((function(e){return e!==i}))}},notifyListeners:function(){for(var e=arguments.length,t=new Array(e),i=0;i<e;i++)t[i]=arguments[i];n.forEach((function(e){return e.apply(void 0,t)}))}}}var b=!("undefined"==typeof window||!window.document||!window.document.createElement);function y(e,n){n(window.confirm(e))}var k="popstate",w="hashchange";function _(){try{return window.history.state||{}}catch(e){return{}}}function x(e){void 0===e&&(e={}),b||Object(c.a)(!1);var n,t=window.history,r=(-1===(n=window.navigator.userAgent).indexOf("Android 2.")&&-1===n.indexOf("Android 4.0")||-1===n.indexOf("Mobile Safari")||-1!==n.indexOf("Chrome")||-1!==n.indexOf("Windows Phone"))&&window.history&&"pushState"in window.history,o=!(-1===window.navigator.userAgent.indexOf("Trident")),a=e,l=a.forceRefresh,s=void 0!==l&&l,p=a.getUserConfirmation,h=void 0===p?y:p,x=a.keyLength,E=void 0===x?6:x,S=e.basename?f(u(e.basename)):"";function T(e){var n=e||{},t=n.key,i=n.state,r=window.location,o=r.pathname+r.search+r.hash;return S&&(o=d(o,S)),g(o,i,t)}function P(){return Math.random().toString(36).substr(2,E)}var C=v();function O(e){Object(i.a)(U,e),U.length=t.length,C.notifyListeners(U.location,U.action)}function A(e){(function(e){return void 0===e.state&&-1===navigator.userAgent.indexOf("CriOS")})(e)||I(T(e.state))}function R(){I(T(_()))}var N=!1;function I(e){if(N)N=!1,O();else{C.confirmTransitionTo(e,"POP",h,(function(n){n?O({action:"POP",location:e}):function(e){var n=U.location,t=D.indexOf(n.key);-1===t&&(t=0);var i=D.indexOf(e.key);-1===i&&(i=0);var r=t-i;r&&(N=!0,M(r))}(e)}))}}var L=T(_()),D=[L.key];function j(e){return S+m(e)}function M(e){t.go(e)}var F=0;function B(e){1===(F+=e)&&1===e?(window.addEventListener(k,A),o&&window.addEventListener(w,R)):0===F&&(window.removeEventListener(k,A),o&&window.removeEventListener(w,R))}var z=!1;var U={length:t.length,action:"POP",location:L,createHref:j,push:function(e,n){var i="PUSH",o=g(e,n,P(),U.location);C.confirmTransitionTo(o,i,h,(function(e){if(e){var n=j(o),a=o.key,l=o.state;if(r)if(t.pushState({key:a,state:l},null,n),s)window.location.href=n;else{var c=D.indexOf(U.location.key),u=D.slice(0,c+1);u.push(o.key),D=u,O({action:i,location:o})}else window.location.href=n}}))},replace:function(e,n){var i="REPLACE",o=g(e,n,P(),U.location);C.confirmTransitionTo(o,i,h,(function(e){if(e){var n=j(o),a=o.key,l=o.state;if(r)if(t.replaceState({key:a,state:l},null,n),s)window.location.replace(n);else{var c=D.indexOf(U.location.key);-1!==c&&(D[c]=o.key),O({action:i,location:o})}else window.location.replace(n)}}))},go:M,goBack:function(){M(-1)},goForward:function(){M(1)},block:function(e){void 0===e&&(e=!1);var n=C.setPrompt(e);return z||(B(1),z=!0),function(){return z&&(z=!1,B(-1)),n()}},listen:function(e){var n=C.appendListener(e);return B(1),function(){B(-1),n()}}};return U}var E="hashchange",S={hashbang:{encodePath:function(e){return"!"===e.charAt(0)?e:"!/"+p(e)},decodePath:function(e){return"!"===e.charAt(0)?e.substr(1):e}},noslash:{encodePath:p,decodePath:u},slash:{encodePath:u,decodePath:u}};function T(e){var n=e.indexOf("#");return-1===n?e:e.slice(0,n)}function P(){var e=window.location.href,n=e.indexOf("#");return-1===n?"":e.substring(n+1)}function C(e){window.location.replace(T(window.location.href)+"#"+e)}function O(e){void 0===e&&(e={}),b||Object(c.a)(!1);var n=window.history,t=(window.navigator.userAgent.indexOf("Firefox"),e),r=t.getUserConfirmation,o=void 0===r?y:r,a=t.hashType,l=void 0===a?"slash":a,s=e.basename?f(u(e.basename)):"",p=S[l],h=p.encodePath,k=p.decodePath;function w(){var e=k(P());return s&&(e=d(e,s)),g(e)}var _=v();function x(e){Object(i.a)(z,e),z.length=n.length,_.notifyListeners(z.location,z.action)}var O=!1,A=null;function R(){var e,n,t=P(),i=h(t);if(t!==i)C(i);else{var r=w(),a=z.location;if(!O&&(n=r,(e=a).pathname===n.pathname&&e.search===n.search&&e.hash===n.hash))return;if(A===m(r))return;A=null,function(e){if(O)O=!1,x();else{var n="POP";_.confirmTransitionTo(e,n,o,(function(t){t?x({action:n,location:e}):function(e){var n=z.location,t=D.lastIndexOf(m(n));-1===t&&(t=0);var i=D.lastIndexOf(m(e));-1===i&&(i=0);var r=t-i;r&&(O=!0,j(r))}(e)}))}}(r)}}var N=P(),I=h(N);N!==I&&C(I);var L=w(),D=[m(L)];function j(e){n.go(e)}var M=0;function F(e){1===(M+=e)&&1===e?window.addEventListener(E,R):0===M&&window.removeEventListener(E,R)}var B=!1;var z={length:n.length,action:"POP",location:L,createHref:function(e){var n=document.querySelector("base"),t="";return n&&n.getAttribute("href")&&(t=T(window.location.href)),t+"#"+h(s+m(e))},push:function(e,n){var t="PUSH",i=g(e,void 0,void 0,z.location);_.confirmTransitionTo(i,t,o,(function(e){if(e){var n=m(i),r=h(s+n);if(P()!==r){A=n,function(e){window.location.hash=e}(r);var o=D.lastIndexOf(m(z.location)),a=D.slice(0,o+1);a.push(n),D=a,x({action:t,location:i})}else x()}}))},replace:function(e,n){var t="REPLACE",i=g(e,void 0,void 0,z.location);_.confirmTransitionTo(i,t,o,(function(e){if(e){var n=m(i),r=h(s+n);P()!==r&&(A=n,C(r));var o=D.indexOf(m(z.location));-1!==o&&(D[o]=n),x({action:t,location:i})}}))},go:j,goBack:function(){j(-1)},goForward:function(){j(1)},block:function(e){void 0===e&&(e=!1);var n=_.setPrompt(e);return B||(F(1),B=!0),function(){return B&&(B=!1,F(-1)),n()}},listen:function(e){var n=_.appendListener(e);return F(1),function(){F(-1),n()}}};return z}function A(e,n,t){return Math.min(Math.max(e,n),t)}function R(e){void 0===e&&(e={});var n=e,t=n.getUserConfirmation,r=n.initialEntries,o=void 0===r?["/"]:r,a=n.initialIndex,l=void 0===a?0:a,s=n.keyLength,c=void 0===s?6:s,u=v();function p(e){Object(i.a)(k,e),k.length=k.entries.length,u.notifyListeners(k.location,k.action)}function d(){return Math.random().toString(36).substr(2,c)}var f=A(l,0,o.length-1),h=o.map((function(e){return g(e,void 0,"string"==typeof e?d():e.key||d())})),b=m;function y(e){var n=A(k.index+e,0,k.entries.length-1),i=k.entries[n];u.confirmTransitionTo(i,"POP",t,(function(e){e?p({action:"POP",location:i,index:n}):p()}))}var k={length:h.length,action:"POP",location:h[f],index:f,entries:h,createHref:b,push:function(e,n){var i="PUSH",r=g(e,n,d(),k.location);u.confirmTransitionTo(r,i,t,(function(e){if(e){var n=k.index+1,t=k.entries.slice(0);t.length>n?t.splice(n,t.length-n,r):t.push(r),p({action:i,location:r,index:n,entries:t})}}))},replace:function(e,n){var i="REPLACE",r=g(e,n,d(),k.location);u.confirmTransitionTo(r,i,t,(function(e){e&&(k.entries[k.index]=r,p({action:i,location:r}))}))},go:y,goBack:function(){y(-1)},goForward:function(){y(1)},canGo:function(e){var n=k.index+e;return n>=0&&n<k.entries.length},block:function(e){return void 0===e&&(e=!1),u.setPrompt(e)},listen:function(e){return u.appendListener(e)}};return k}},function(e,n,t){"use strict";function i(e,n){if(null==e)return{};var t,i,r={},o=Object.keys(e);for(i=0;i<o.length;i++)t=o[i],n.indexOf(t)>=0||(r[t]=e[t]);return r}t.d(n,"a",(function(){return i}))},function(e,n,t){"use strict";var i=!("undefined"==typeof window||!window.document||!window.document.createElement),r={canUseDOM:i,canUseEventListeners:i&&!(!window.addEventListener&&!window.attachEvent),canUseIntersectionObserver:i&&"IntersectionObserver"in window,canUseViewport:i&&!!window.screen};n.a=r},function(e,n,t){"use strict";t.d(n,"a",(function(){return l})),t.d(n,"b",(function(){return s}));var i=t(2),r=t(3),o=t(0),a=t.n(o);function l(e,n,t){return void 0===t&&(t=[]),e.some((function(e){var r=e.path?Object(i.j)(n,e):t.length?t[t.length-1].match:i.e.computeRootMatch(n);return r&&(t.push({route:e,match:r}),e.routes&&l(e.routes,n,t)),r})),t}function s(e,n,t){return void 0===n&&(n={}),void 0===t&&(t={}),e?a.a.createElement(i.g,t,e.map((function(e,t){return a.a.createElement(i.d,{key:e.key||t,path:e.path,exact:e.exact,strict:e.strict,render:function(t){return e.render?e.render(Object(r.a)({},t,{},n,{route:e})):a.a.createElement(e.component,Object(r.a)({},t,n,{route:e}))}})}))):null}},function(e,n,t){"use strict";t.d(n,"a",(function(){return p})),t.d(n,"b",(function(){return d})),t.d(n,"c",(function(){return b})),t.d(n,"e",(function(){return w}));var i=t(2);t.d(n,"d",(function(){return i.a})),t.d(n,"f",(function(){return i.b})),t.d(n,"g",(function(){return i.c})),t.d(n,"h",(function(){return i.d})),t.d(n,"i",(function(){return i.e})),t.d(n,"j",(function(){return i.f})),t.d(n,"k",(function(){return i.g})),t.d(n,"l",(function(){return i.i})),t.d(n,"m",(function(){return i.j})),t.d(n,"n",(function(){return i.k})),t.d(n,"o",(function(){return i.l})),t.d(n,"p",(function(){return i.m})),t.d(n,"q",(function(){return i.n})),t.d(n,"r",(function(){return i.o}));var r=t(4),o=t(0),a=t.n(o),l=t(6),s=t(3),c=t(7),u=t(5),p=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(l.a)(n.props),n}return Object(r.a)(n,e),n.prototype.render=function(){return a.a.createElement(i.e,{history:this.history,children:this.props.children})},n}(a.a.Component);var d=function(e){function n(){for(var n,t=arguments.length,i=new Array(t),r=0;r<t;r++)i[r]=arguments[r];return(n=e.call.apply(e,[this].concat(i))||this).history=Object(l.b)(n.props),n}return Object(r.a)(n,e),n.prototype.render=function(){return a.a.createElement(i.e,{history:this.history,children:this.props.children})},n}(a.a.Component);var f=function(e,n){return"function"==typeof e?e(n):e},m=function(e,n){return"string"==typeof e?Object(l.c)(e,null,null,n):e},g=function(e){return e},h=a.a.forwardRef;void 0===h&&(h=g);var v=h((function(e,n){var t=e.innerRef,i=e.navigate,r=e.onClick,o=Object(c.a)(e,["innerRef","navigate","onClick"]),l=o.target,u=Object(s.a)({},o,{onClick:function(e){try{r&&r(e)}catch(n){throw e.preventDefault(),n}e.defaultPrevented||0!==e.button||l&&"_self"!==l||function(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}(e)||(e.preventDefault(),i())}});return u.ref=g!==h&&n||t,a.a.createElement("a",u)}));var b=h((function(e,n){var t=e.component,r=void 0===t?v:t,o=e.replace,l=e.to,p=e.innerRef,d=Object(c.a)(e,["component","replace","to","innerRef"]);return a.a.createElement(i.h.Consumer,null,(function(e){e||Object(u.a)(!1);var t=e.history,i=m(f(l,e.location),e.location),c=i?t.createHref(i):"",v=Object(s.a)({},d,{href:c,navigate:function(){var n=f(l,e.location);(o?t.replace:t.push)(n)}});return g!==h?v.ref=n||p:v.innerRef=p,a.a.createElement(r,v)}))})),y=function(e){return e},k=a.a.forwardRef;void 0===k&&(k=y);var w=k((function(e,n){var t=e["aria-current"],r=void 0===t?"page":t,o=e.activeClassName,l=void 0===o?"active":o,p=e.activeStyle,d=e.className,g=e.exact,h=e.isActive,v=e.location,w=e.sensitive,_=e.strict,x=e.style,E=e.to,S=e.innerRef,T=Object(c.a)(e,["aria-current","activeClassName","activeStyle","className","exact","isActive","location","sensitive","strict","style","to","innerRef"]);return a.a.createElement(i.h.Consumer,null,(function(e){e||Object(u.a)(!1);var t=v||e.location,o=m(f(E,t),t),c=o.pathname,P=c&&c.replace(/([.+*?=^!:${}()[\]|/\\])/g,"\\$1"),C=P?Object(i.j)(t.pathname,{path:P,exact:g,sensitive:w,strict:_}):null,O=!!(h?h(C,t):C),A=O?function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];return n.filter((function(e){return e})).join(" ")}(d,l):d,R=O?Object(s.a)({},x,{},p):x,N=Object(s.a)({"aria-current":O&&r||null,className:A,style:R,to:o},T);return y!==k?N.ref=n||S:N.innerRef=S,a.a.createElement(b,N)}))}))},function(e,n,t){"use strict";t.r(n),n.default={title:"Netsage Pipeline Documentation",tagline:"Netsage Pipeline Documentation",url:"https://netsage-project.github.io",baseUrl:"/netsage-pipeline/",favicon:"img/favicon.ico",organizationName:"netsage-project",projectName:"netsage-pipeline",themeConfig:{navbar:{title:"NetSage Documentation",logo:{alt:"NetSage Logo",src:"img/logo.png"},items:[{label:"Pipeline Documentation",position:"left",activeBasePath:"docs",to:"docs/pipeline/intro"},{href:"https://netsage-project.github.io/netsage-grafana-configs/",label:"Dashboard Documentation",position:"left",target:"_self"},{type:"docsVersionDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{href:"https://github.com/netsage-project/netsage-pipeline/",label:"GitHub",position:"right"}],hideOnScroll:!1},footer:{style:"dark",links:[],copyright:"Copyright \xa9 2023 NetSage Built with Docusaurus."},colorMode:{defaultMode:"light",disableSwitch:!1,respectPrefersColorScheme:!1,switchConfig:{darkIcon:"\ud83c\udf1c",darkIconStyle:{},lightIcon:"\ud83c\udf1e",lightIconStyle:{}}},docs:{versionPersistence:"localStorage"},metadatas:[],prism:{additionalLanguages:[]},hideableSidebar:!1},presets:[["@docusaurus/preset-classic",{docs:{remarkPlugins:[null],sidebarPath:"/home/dsouthworth/netsage-pipeline/website/sidebars.js",lastVersion:"2.0.0",versions:{current:{label:"master (unreleased)"}},editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/website/"},blog:{showReadingTime:!0,editUrl:"https://github.com/netsage-project/netsage-pipeline/edit/master/blog/"},theme:{customCss:"/home/dsouthworth/netsage-pipeline/website/src/css/custom.css"}}]],baseUrlIssueBanner:!0,i18n:{defaultLocale:"en",locales:["en"],localeConfigs:{}},onBrokenLinks:"throw",onBrokenMarkdownLinks:"warn",onDuplicateRoutes:"warn",customFields:{},plugins:[],themes:[],titleDelimiter:"|",noIndex:!1}},function(e,n,t){"use strict";var i=Object.getOwnPropertySymbols,r=Object.prototype.hasOwnProperty,o=Object.prototype.propertyIsEnumerable;function a(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var n={},t=0;t<10;t++)n["_"+String.fromCharCode(t)]=t;if("0123456789"!==Object.getOwnPropertyNames(n).map((function(e){return n[e]})).join(""))return!1;var i={};return"abcdefghijklmnopqrst".split("").forEach((function(e){i[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},i)).join("")}catch(r){return!1}}()?Object.assign:function(e,n){for(var t,l,s=a(e),c=1;c<arguments.length;c++){for(var u in t=Object(arguments[c]))r.call(t,u)&&(s[u]=t[u]);if(i){l=i(t);for(var p=0;p<l.length;p++)o.call(t,l[p])&&(s[l[p]]=t[l[p]])}}return s}},function(e){e.exports=JSON.parse('{"/netsage-pipeline/-421":{"component":"c4f5d8e4","config":"5e9f5e1a"},"/netsage-pipeline/docs/1.2.10-296":{"component":"1be78505","versionMetadata":"d5087a67"},"/netsage-pipeline/docs/1.2.10/components/docker_env-63f":{"component":"17896441","content":"e10970d8"},"/netsage-pipeline/docs/1.2.10/components/docker_first_steps-059":{"component":"17896441","content":"ae8a1364"},"/netsage-pipeline/docs/1.2.10/components/docker_pipeline-a0b":{"component":"17896441","content":"46a600e4"},"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install-ec8":{"component":"17896441","content":"e7670743"},"/netsage-pipeline/docs/1.2.10/deploy/choose_install-fb2":{"component":"17896441","content":"8bbfcfa9"},"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced-a83":{"component":"17896441","content":"f1ddb0ed"},"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple-15f":{"component":"17896441","content":"e746408e"},"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot-e2e":{"component":"17896441","content":"cd902920"},"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade-b01":{"component":"17896441","content":"38962575"},"/netsage-pipeline/docs/1.2.10/devel/dev_dataset-7fa":{"component":"17896441","content":"f485eca7"},"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide-f40":{"component":"17896441","content":"63f5fe43"},"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag-b8f":{"component":"17896441","content":"b83a1049"},"/netsage-pipeline/docs/1.2.10/devel/docusaurus-bcf":{"component":"17896441","content":"514d766a"},"/netsage-pipeline/docs/1.2.10/pipeline/elastic-c07":{"component":"17896441","content":"4a74d8e8"},"/netsage-pipeline/docs/1.2.10/pipeline/importer-3c0":{"component":"17896441","content":"799b1578"},"/netsage-pipeline/docs/1.2.10/pipeline/intro-3b3":{"component":"17896441","content":"49e74cdd"},"/netsage-pipeline/docs/1.2.10/pipeline/logstash-116":{"component":"17896441","content":"0271802f"},"/netsage-pipeline/docs/1.2.10/pipeline/nfdump-f4d":{"component":"17896441","content":"f8bc0717"},"/netsage-pipeline/docs/1.2.10/pipeline/tstat-4c2":{"component":"17896441","content":"0b8e461a"},"/netsage-pipeline/docs/1.2.11-d04":{"component":"1be78505","versionMetadata":"02341ef2"},"/netsage-pipeline/docs/1.2.11/components/docker_env-1d6":{"component":"17896441","content":"8997819f"},"/netsage-pipeline/docs/1.2.11/components/docker_first_steps-62e":{"component":"17896441","content":"e987ec87"},"/netsage-pipeline/docs/1.2.11/components/docker_pipeline-6b1":{"component":"17896441","content":"69c1008e"},"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install-c46":{"component":"17896441","content":"0327c8bf"},"/netsage-pipeline/docs/1.2.11/deploy/choose_install-f72":{"component":"17896441","content":"05c6fd0e"},"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced-0a8":{"component":"17896441","content":"22d86c11"},"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple-c8a":{"component":"17896441","content":"dd1a9f26"},"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot-a22":{"component":"17896441","content":"7c78db09"},"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade-9ae":{"component":"17896441","content":"d48b255e"},"/netsage-pipeline/docs/1.2.11/devel/dev_dataset-ba4":{"component":"17896441","content":"9a5c9747"},"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide-80d":{"component":"17896441","content":"41980e0e"},"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag-681":{"component":"17896441","content":"6d913b5d"},"/netsage-pipeline/docs/1.2.11/devel/docusaurus-e33":{"component":"17896441","content":"9d5eac8e"},"/netsage-pipeline/docs/1.2.11/pipeline/elastic-ef5":{"component":"17896441","content":"5b6c4cb7"},"/netsage-pipeline/docs/1.2.11/pipeline/importer-5cc":{"component":"17896441","content":"96852331"},"/netsage-pipeline/docs/1.2.11/pipeline/intro-5fd":{"component":"17896441","content":"a12215ea"},"/netsage-pipeline/docs/1.2.11/pipeline/logstash-0d3":{"component":"17896441","content":"012f72bc"},"/netsage-pipeline/docs/1.2.11/pipeline/nfdump-9b4":{"component":"17896441","content":"e30bbbcf"},"/netsage-pipeline/docs/1.2.11/pipeline/tstat-3cb":{"component":"17896441","content":"f52ba0ac"},"/netsage-pipeline/docs/1.2.5-c52":{"component":"1be78505","versionMetadata":"6c279c12"},"/netsage-pipeline/docs/1.2.5/deploy/choose_install-3ce":{"component":"17896441","content":"3179f6dc"},"/netsage-pipeline/docs/1.2.5/deploy/docker_install-d98":{"component":"17896441","content":"a04bc459"},"/netsage-pipeline/docs/1.2.5/deploy/install-47c":{"component":"17896441","content":"9fe55709"},"/netsage-pipeline/docs/1.2.5/devel/docker-0e4":{"component":"17896441","content":"83dbc9a5"},"/netsage-pipeline/docs/1.2.5/docusaurus-882":{"component":"17896441","content":"480c5911"},"/netsage-pipeline/docs/1.2.5/pipeline-8ff":{"component":"17896441","content":"c0bd86e8"},"/netsage-pipeline/docs/1.2.5/pipeline_importer-93b":{"component":"17896441","content":"066f8364"},"/netsage-pipeline/docs/1.2.5/pipeline_logstash-9ed":{"component":"17896441","content":"a7459da2"},"/netsage-pipeline/docs/1.2.6-489":{"component":"1be78505","versionMetadata":"7d380c4c"},"/netsage-pipeline/docs/1.2.6/deploy/choose_install-0d1":{"component":"17896441","content":"d5479f42"},"/netsage-pipeline/docs/1.2.6/deploy/docker_install-4df":{"component":"17896441","content":"9feaf8ab"},"/netsage-pipeline/docs/1.2.6/deploy/install-8ee":{"component":"17896441","content":"e2be698d"},"/netsage-pipeline/docs/1.2.6/devel/docker-d95":{"component":"17896441","content":"aa98ec4a"},"/netsage-pipeline/docs/1.2.6/docusaurus-312":{"component":"17896441","content":"d91f9def"},"/netsage-pipeline/docs/1.2.6/pipeline-7ea":{"component":"17896441","content":"ebfaece6"},"/netsage-pipeline/docs/1.2.6/pipeline_importer-c76":{"component":"17896441","content":"21ee0bc4"},"/netsage-pipeline/docs/1.2.6/pipeline_logstash-ee6":{"component":"17896441","content":"38d1fe39"},"/netsage-pipeline/docs/1.2.7-3e0":{"component":"1be78505","versionMetadata":"906738b8"},"/netsage-pipeline/docs/1.2.7/components/docker_env-fb9":{"component":"17896441","content":"358e6759"},"/netsage-pipeline/docs/1.2.7/components/docker_pipeline-28e":{"component":"17896441","content":"e436ce66"},"/netsage-pipeline/docs/1.2.7/components/docker_upgrade-aca":{"component":"17896441","content":"4ff24bf8"},"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install-88b":{"component":"17896441","content":"aafd28b2"},"/netsage-pipeline/docs/1.2.7/deploy/choose_install-9c2":{"component":"17896441","content":"5c4beb71"},"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced-a68":{"component":"17896441","content":"ac3b7d69"},"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple-c15":{"component":"17896441","content":"879923b2"},"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot-f5e":{"component":"17896441","content":"d71ff009"},"/netsage-pipeline/docs/1.2.7/devel/dev_dataset-c2f":{"component":"17896441","content":"3914abfa"},"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide-13c":{"component":"17896441","content":"97edd56e"},"/netsage-pipeline/docs/1.2.7/docusaurus-fba":{"component":"17896441","content":"0381afdd"},"/netsage-pipeline/docs/1.2.7/pipeline/elastic-545":{"component":"17896441","content":"3309db24"},"/netsage-pipeline/docs/1.2.7/pipeline/importer-457":{"component":"17896441","content":"43862f47"},"/netsage-pipeline/docs/1.2.7/pipeline/intro-d5a":{"component":"17896441","content":"07faddd5"},"/netsage-pipeline/docs/1.2.7/pipeline/logstash-144":{"component":"17896441","content":"bb237b1e"},"/netsage-pipeline/docs/1.2.7/pipeline/nfdump-6d8":{"component":"17896441","content":"42f7ba93"},"/netsage-pipeline/docs/1.2.7/pipeline/tstat-d6c":{"component":"17896441","content":"e186625e"},"/netsage-pipeline/docs/1.2.8-51d":{"component":"1be78505","versionMetadata":"992819d4"},"/netsage-pipeline/docs/1.2.8/components/docker_env-f3f":{"component":"17896441","content":"abbf3a21"},"/netsage-pipeline/docs/1.2.8/components/docker_first_steps-d25":{"component":"17896441","content":"9bfdba8c"},"/netsage-pipeline/docs/1.2.8/components/docker_pipeline-7d7":{"component":"17896441","content":"42f0e773"},"/netsage-pipeline/docs/1.2.8/components/docker_upgrade-db5":{"component":"17896441","content":"a55d7fc8"},"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install-a0d":{"component":"17896441","content":"49703371"},"/netsage-pipeline/docs/1.2.8/deploy/choose_install-298":{"component":"17896441","content":"d2c4f39e"},"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced-150":{"component":"17896441","content":"56776a50"},"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple-f57":{"component":"17896441","content":"42e094bc"},"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot-a18":{"component":"17896441","content":"d5140579"},"/netsage-pipeline/docs/1.2.8/devel/dev_dataset-744":{"component":"17896441","content":"e00e17a0"},"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide-e12":{"component":"17896441","content":"0214960a"},"/netsage-pipeline/docs/1.2.8/docusaurus-57f":{"component":"17896441","content":"58d958f4"},"/netsage-pipeline/docs/1.2.8/pipeline/elastic-e1a":{"component":"17896441","content":"bed7ab4e"},"/netsage-pipeline/docs/1.2.8/pipeline/importer-b38":{"component":"17896441","content":"ad308aff"},"/netsage-pipeline/docs/1.2.8/pipeline/intro-c0b":{"component":"17896441","content":"33f6b1c9"},"/netsage-pipeline/docs/1.2.8/pipeline/logstash-00d":{"component":"17896441","content":"6a059f5b"},"/netsage-pipeline/docs/1.2.8/pipeline/nfdump-6f1":{"component":"17896441","content":"b1bfe9f5"},"/netsage-pipeline/docs/1.2.8/pipeline/tstat-59a":{"component":"17896441","content":"832ca04e"},"/netsage-pipeline/docs/1.2.9-cb5":{"component":"1be78505","versionMetadata":"d75898ab"},"/netsage-pipeline/docs/1.2.9/components/docker_env-e31":{"component":"17896441","content":"6125e2e5"},"/netsage-pipeline/docs/1.2.9/components/docker_first_steps-253":{"component":"17896441","content":"b3c114f4"},"/netsage-pipeline/docs/1.2.9/components/docker_pipeline-6e2":{"component":"17896441","content":"d1597237"},"/netsage-pipeline/docs/1.2.9/components/docker_upgrade-8be":{"component":"17896441","content":"6546ef57"},"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install-31c":{"component":"17896441","content":"56a29d1a"},"/netsage-pipeline/docs/1.2.9/deploy/choose_install-5ff":{"component":"17896441","content":"f64b3473"},"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced-711":{"component":"17896441","content":"5ee17acf"},"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple-536":{"component":"17896441","content":"c51f4c72"},"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot-7d0":{"component":"17896441","content":"1d038b8a"},"/netsage-pipeline/docs/1.2.9/devel/dev_dataset-e4a":{"component":"17896441","content":"40b960a4"},"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide-b39":{"component":"17896441","content":"9b505a1f"},"/netsage-pipeline/docs/1.2.9/docusaurus-bed":{"component":"17896441","content":"db0ed613"},"/netsage-pipeline/docs/1.2.9/pipeline/elastic-bd5":{"component":"17896441","content":"5a20f281"},"/netsage-pipeline/docs/1.2.9/pipeline/importer-90f":{"component":"17896441","content":"99d6017e"},"/netsage-pipeline/docs/1.2.9/pipeline/intro-e91":{"component":"17896441","content":"c3f49368"},"/netsage-pipeline/docs/1.2.9/pipeline/logstash-9e9":{"component":"17896441","content":"9720d299"},"/netsage-pipeline/docs/1.2.9/pipeline/nfdump-220":{"component":"17896441","content":"7b85b03c"},"/netsage-pipeline/docs/1.2.9/pipeline/tstat-341":{"component":"17896441","content":"50032990"},"/netsage-pipeline/docs/next-8b0":{"component":"1be78505","versionMetadata":"935f2afb"},"/netsage-pipeline/docs/next/components/docker_env-cfe":{"component":"17896441","content":"64269ed8"},"/netsage-pipeline/docs/next/components/docker_first_steps-e72":{"component":"17896441","content":"080a21b4"},"/netsage-pipeline/docs/next/components/docker_pipeline-5eb":{"component":"17896441","content":"ba55e963"},"/netsage-pipeline/docs/next/deploy/bare_metal_install-141":{"component":"17896441","content":"e5ce8f83"},"/netsage-pipeline/docs/next/deploy/choose_install-d51":{"component":"17896441","content":"66e31e6e"},"/netsage-pipeline/docs/next/deploy/docker_install_advanced-de0":{"component":"17896441","content":"cbd1c0d0"},"/netsage-pipeline/docs/next/deploy/docker_install_simple-72d":{"component":"17896441","content":"734dc415"},"/netsage-pipeline/docs/next/deploy/docker_troubleshoot-593":{"component":"17896441","content":"653602f1"},"/netsage-pipeline/docs/next/deploy/docker_upgrade-68f":{"component":"17896441","content":"ceb45c47"},"/netsage-pipeline/docs/next/devel/dev_dataset-091":{"component":"17896441","content":"fa16f0fc"},"/netsage-pipeline/docs/next/devel/docker_dev_guide-85f":{"component":"17896441","content":"1f3a3353"},"/netsage-pipeline/docs/next/devel/docker_dev_tag-e93":{"component":"17896441","content":"16fcefd4"},"/netsage-pipeline/docs/next/devel/docusaurus-702":{"component":"17896441","content":"4ffdf56a"},"/netsage-pipeline/docs/next/pipeline/elastic-533":{"component":"17896441","content":"7d0b7755"},"/netsage-pipeline/docs/next/pipeline/importer-063":{"component":"17896441","content":"d3556f45"},"/netsage-pipeline/docs/next/pipeline/intro-f04":{"component":"17896441","content":"22b7278f"},"/netsage-pipeline/docs/next/pipeline/logstash-aec":{"component":"17896441","content":"bd6c6f88"},"/netsage-pipeline/docs/next/pipeline/nfdump-543":{"component":"17896441","content":"1cc43674"},"/netsage-pipeline/docs/next/pipeline/tstat-474":{"component":"17896441","content":"e2c161d9"},"/netsage-pipeline/docs-89b":{"component":"1be78505","versionMetadata":"9aa54785"},"/netsage-pipeline/docs/components/docker_env-74f":{"component":"17896441","content":"9b6d3056"},"/netsage-pipeline/docs/components/docker_first_steps-c20":{"component":"17896441","content":"a3c17fac"},"/netsage-pipeline/docs/components/docker_pipeline-2f6":{"component":"17896441","content":"89d5514b"},"/netsage-pipeline/docs/deploy/bare_metal_install-faa":{"component":"17896441","content":"b56f03c6"},"/netsage-pipeline/docs/deploy/choose_install-e79":{"component":"17896441","content":"d2c255b4"},"/netsage-pipeline/docs/deploy/docker_install_advanced-161":{"component":"17896441","content":"00cb704f"},"/netsage-pipeline/docs/deploy/docker_install_simple-64b":{"component":"17896441","content":"418618b7"},"/netsage-pipeline/docs/deploy/docker_troubleshoot-bfd":{"component":"17896441","content":"1bbe794e"},"/netsage-pipeline/docs/deploy/docker_upgrade-732":{"component":"17896441","content":"e3649aa7"},"/netsage-pipeline/docs/devel/dev_dataset-4f7":{"component":"17896441","content":"ded423ee"},"/netsage-pipeline/docs/devel/docker_dev_guide-e38":{"component":"17896441","content":"53b0b5d8"},"/netsage-pipeline/docs/devel/docker_dev_tag-194":{"component":"17896441","content":"e4c61278"},"/netsage-pipeline/docs/devel/docusaurus-039":{"component":"17896441","content":"0d6d53ac"},"/netsage-pipeline/docs/pipeline/elastic-023":{"component":"17896441","content":"9001d127"},"/netsage-pipeline/docs/pipeline/importer-91a":{"component":"17896441","content":"d1b2eded"},"/netsage-pipeline/docs/pipeline/intro-c00":{"component":"17896441","content":"af9cfdf8"},"/netsage-pipeline/docs/pipeline/logstash-105":{"component":"17896441","content":"9e1ec80c"},"/netsage-pipeline/docs/pipeline/nfdump-d28":{"component":"17896441","content":"59305d05"},"/netsage-pipeline/docs/pipeline/tstat-c1d":{"component":"17896441","content":"8084fbd4"}}')},function(e,n,t){"use strict";var i=t(0),r=t.n(i);n.a=r.a.createContext(null)},function(e,n,t){var i,r;void 0===(r="function"==typeof(i=function(){var e,n,t={version:"0.2.0"},i=t.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'<div class="bar" role="bar"><div class="peg"></div></div><div class="spinner" role="spinner"><div class="spinner-icon"></div></div>'};function r(e,n,t){return e<n?n:e>t?t:e}function o(e){return 100*(-1+e)}function a(e,n,t){var r;return(r="translate3d"===i.positionUsing?{transform:"translate3d("+o(e)+"%,0,0)"}:"translate"===i.positionUsing?{transform:"translate("+o(e)+"%,0)"}:{"margin-left":o(e)+"%"}).transition="all "+n+"ms "+t,r}t.configure=function(e){var n,t;for(n in e)void 0!==(t=e[n])&&e.hasOwnProperty(n)&&(i[n]=t);return this},t.status=null,t.set=function(e){var n=t.isStarted();e=r(e,i.minimum,1),t.status=1===e?null:e;var o=t.render(!n),c=o.querySelector(i.barSelector),u=i.speed,p=i.easing;return o.offsetWidth,l((function(n){""===i.positionUsing&&(i.positionUsing=t.getPositioningCSS()),s(c,a(e,u,p)),1===e?(s(o,{transition:"none",opacity:1}),o.offsetWidth,setTimeout((function(){s(o,{transition:"all "+u+"ms linear",opacity:0}),setTimeout((function(){t.remove(),n()}),u)}),u)):setTimeout(n,u)})),this},t.isStarted=function(){return"number"==typeof t.status},t.start=function(){t.status||t.set(0);var e=function(){setTimeout((function(){t.status&&(t.trickle(),e())}),i.trickleSpeed)};return i.trickle&&e(),this},t.done=function(e){return e||t.status?t.inc(.3+.5*Math.random()).set(1):this},t.inc=function(e){var n=t.status;return n?("number"!=typeof e&&(e=(1-n)*r(Math.random()*n,.1,.95)),n=r(n+e,0,.994),t.set(n)):t.start()},t.trickle=function(){return t.inc(Math.random()*i.trickleRate)},e=0,n=0,t.promise=function(i){return i&&"resolved"!==i.state()?(0===n&&t.start(),e++,n++,i.always((function(){0==--n?(e=0,t.done()):t.set((e-n)/e)})),this):this},t.render=function(e){if(t.isRendered())return document.getElementById("nprogress");u(document.documentElement,"nprogress-busy");var n=document.createElement("div");n.id="nprogress",n.innerHTML=i.template;var r,a=n.querySelector(i.barSelector),l=e?"-100":o(t.status||0),c=document.querySelector(i.parent);return s(a,{transition:"all 0 linear",transform:"translate3d("+l+"%,0,0)"}),i.showSpinner||(r=n.querySelector(i.spinnerSelector))&&f(r),c!=document.body&&u(c,"nprogress-custom-parent"),c.appendChild(n),n},t.remove=function(){p(document.documentElement,"nprogress-busy"),p(document.querySelector(i.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&f(e)},t.isRendered=function(){return!!document.getElementById("nprogress")},t.getPositioningCSS=function(){var e=document.body.style,n="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return n+"Perspective"in e?"translate3d":n+"Transform"in e?"translate":"margin"};var l=function(){var e=[];function n(){var t=e.shift();t&&t(n)}return function(t){e.push(t),1==e.length&&n()}}(),s=function(){var e=["Webkit","O","Moz","ms"],n={};function t(e){return e.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,(function(e,n){return n.toUpperCase()}))}function i(n){var t=document.body.style;if(n in t)return n;for(var i,r=e.length,o=n.charAt(0).toUpperCase()+n.slice(1);r--;)if((i=e[r]+o)in t)return i;return n}function r(e){return e=t(e),n[e]||(n[e]=i(e))}function o(e,n,t){n=r(n),e.style[n]=t}return function(e,n){var t,i,r=arguments;if(2==r.length)for(t in n)void 0!==(i=n[t])&&n.hasOwnProperty(t)&&o(e,t,i);else o(e,r[1],r[2])}}();function c(e,n){return("string"==typeof e?e:d(e)).indexOf(" "+n+" ")>=0}function u(e,n){var t=d(e),i=t+n;c(t,n)||(e.className=i.substring(1))}function p(e,n){var t,i=d(e);c(e,n)&&(t=i.replace(" "+n+" "," "),e.className=t.substring(1,t.length-1))}function d(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function f(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return t})?i.call(n,t,n,e):i)||(e.exports=r)},function(e,n,t){"use strict";t.r(n);var i=t(0),r=t(14);n.default=function(){var e=Object(i.useContext)(r.a);if(null===e)throw new Error("Docusaurus context not provided");return e}},,,function(e,n,t){var i={"./":48};function r(e){var n=o(e);return t(n)}function o(e){if(!t.o(i,e)){var n=new Error("Cannot find module '"+e+"'");throw n.code="MODULE_NOT_FOUND",n}return i[e]}r.keys=function(){return Object.keys(i)},r.resolve=o,e.exports=r,r.id=19},function(e,n,t){"use strict";function i(e,n){e.prototype=Object.create(n.prototype),e.prototype.constructor=e,e.__proto__=n}function r(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(){return(a=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e}).apply(this,arguments)}var l=t(0),s=t(1),c=[],u=[];function p(e){var n=e(),t={loading:!0,loaded:null,error:null};return t.promise=n.then((function(e){return t.loading=!1,t.loaded=e,e})).catch((function(e){throw t.loading=!1,t.error=e,e})),t}function d(e){var n={loading:!1,loaded:{},error:null},t=[];try{Object.keys(e).forEach((function(i){var r=p(e[i]);r.loading?n.loading=!0:(n.loaded[i]=r.loaded,n.error=r.error),t.push(r.promise),r.promise.then((function(e){n.loaded[i]=e})).catch((function(e){n.error=e}))}))}catch(i){n.error=i}return n.promise=Promise.all(t).then((function(e){return n.loading=!1,e})).catch((function(e){throw n.loading=!1,e})),n}function f(e,n){return l.createElement((t=e)&&t.__esModule?t.default:t,n);var t}function m(e,n){var p,d;if(!n.loading)throw new Error("react-loadable requires a `loading` component");var m=a({loader:null,loading:null,delay:200,timeout:null,render:f,webpack:null,modules:null},n),g=null;function h(){return g||(g=e(m.loader)),g.promise}return c.push(h),"function"==typeof m.webpack&&u.push((function(){if((0,m.webpack)().every((function(e){return void 0!==e&&void 0!==t.m[e]})))return h()})),d=p=function(n){function t(t){var i;return o(r(r(i=n.call(this,t)||this)),"retry",(function(){i.setState({error:null,loading:!0,timedOut:!1}),g=e(m.loader),i._loadModule()})),h(),i.state={error:g.error,pastDelay:!1,timedOut:!1,loading:g.loading,loaded:g.loaded},i}i(t,n),t.preload=function(){return h()};var a=t.prototype;return a.UNSAFE_componentWillMount=function(){this._loadModule()},a.componentDidMount=function(){this._mounted=!0},a._loadModule=function(){var e=this;if(this.context.loadable&&Array.isArray(m.modules)&&m.modules.forEach((function(n){e.context.loadable.report(n)})),g.loading){var n=function(n){e._mounted&&e.setState(n)};"number"==typeof m.delay&&(0===m.delay?this.setState({pastDelay:!0}):this._delay=setTimeout((function(){n({pastDelay:!0})}),m.delay)),"number"==typeof m.timeout&&(this._timeout=setTimeout((function(){n({timedOut:!0})}),m.timeout));var t=function(){n({error:g.error,loaded:g.loaded,loading:g.loading}),e._clearTimeouts()};g.promise.then((function(){return t(),null})).catch((function(e){return t(),null}))}},a.componentWillUnmount=function(){this._mounted=!1,this._clearTimeouts()},a._clearTimeouts=function(){clearTimeout(this._delay),clearTimeout(this._timeout)},a.render=function(){return this.state.loading||this.state.error?l.createElement(m.loading,{isLoading:this.state.loading,pastDelay:this.state.pastDelay,timedOut:this.state.timedOut,error:this.state.error,retry:this.retry}):this.state.loaded?m.render(this.state.loaded,this.props):null},t}(l.Component),o(p,"contextTypes",{loadable:s.shape({report:s.func.isRequired})}),d}function g(e){return m(p,e)}g.Map=function(e){if("function"!=typeof e.render)throw new Error("LoadableMap requires a `render(loaded, props)` function");return m(d,e)};var h=function(e){function n(){return e.apply(this,arguments)||this}i(n,e);var t=n.prototype;return t.getChildContext=function(){return{loadable:{report:this.props.report}}},t.render=function(){return l.Children.only(this.props.children)},n}(l.Component);function v(e){for(var n=[];e.length;){var t=e.pop();n.push(t())}return Promise.all(n).then((function(){if(e.length)return v(e)}))}o(h,"propTypes",{report:s.func.isRequired}),o(h,"childContextTypes",{loadable:s.shape({report:s.func.isRequired}).isRequired}),g.Capture=h,g.preloadAll=function(){return new Promise((function(e,n){v(c).then(e,n)}))},g.preloadReady=function(){return new Promise((function(e,n){v(u).then(e,e)}))},e.exports=g},function(e,n,t){"use strict";(function(e){var i=t(0),r=t.n(i),o=t(4),a=t(1),l=t.n(a),s=1073741823,c="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:void 0!==e?e:{};function u(e){var n=[];return{on:function(e){n.push(e)},off:function(e){n=n.filter((function(n){return n!==e}))},get:function(){return e},set:function(t,i){e=t,n.forEach((function(n){return n(e,i)}))}}}var p=r.a.createContext||function(e,n){var t,r,a,p="__create-react-context-"+((c[a="__global_unique_id__"]=(c[a]||0)+1)+"__"),d=function(e){function t(){var n;return(n=e.apply(this,arguments)||this).emitter=u(n.props.value),n}Object(o.a)(t,e);var i=t.prototype;return i.getChildContext=function(){var e;return(e={})[p]=this.emitter,e},i.componentWillReceiveProps=function(e){if(this.props.value!==e.value){var t,i=this.props.value,r=e.value;((o=i)===(a=r)?0!==o||1/o==1/a:o!=o&&a!=a)?t=0:(t="function"==typeof n?n(i,r):s,0!==(t|=0)&&this.emitter.set(e.value,t))}var o,a},i.render=function(){return this.props.children},t}(i.Component);d.childContextTypes=((t={})[p]=l.a.object.isRequired,t);var f=function(n){function t(){var e;return(e=n.apply(this,arguments)||this).state={value:e.getValue()},e.onUpdate=function(n,t){0!=((0|e.observedBits)&t)&&e.setState({value:e.getValue()})},e}Object(o.a)(t,n);var i=t.prototype;return i.componentWillReceiveProps=function(e){var n=e.observedBits;this.observedBits=null==n?s:n},i.componentDidMount=function(){this.context[p]&&this.context[p].on(this.onUpdate);var e=this.props.observedBits;this.observedBits=null==e?s:e},i.componentWillUnmount=function(){this.context[p]&&this.context[p].off(this.onUpdate)},i.getValue=function(){return this.context[p]?this.context[p].get():e},i.render=function(){return(e=this.props.children,Array.isArray(e)?e[0]:e)(this.state.value);var e},t}(i.Component);return f.contextTypes=((r={})[p]=l.a.object,r),{Provider:d,Consumer:f}};n.a=p}).call(this,t(28))},function(e,n,t){var i=t(45);e.exports=f,e.exports.parse=o,e.exports.compile=function(e,n){return l(o(e,n),n)},e.exports.tokensToFunction=l,e.exports.tokensToRegExp=d;var r=new RegExp(["(\\\\.)","([\\/.])?(?:(?:\\:(\\w+)(?:\\(((?:\\\\.|[^\\\\()])+)\\))?|\\(((?:\\\\.|[^\\\\()])+)\\))([+*?])?|(\\*))"].join("|"),"g");function o(e,n){for(var t,i=[],o=0,a=0,l="",u=n&&n.delimiter||"/";null!=(t=r.exec(e));){var p=t[0],d=t[1],f=t.index;if(l+=e.slice(a,f),a=f+p.length,d)l+=d[1];else{var m=e[a],g=t[2],h=t[3],v=t[4],b=t[5],y=t[6],k=t[7];l&&(i.push(l),l="");var w=null!=g&&null!=m&&m!==g,_="+"===y||"*"===y,x="?"===y||"*"===y,E=t[2]||u,S=v||b;i.push({name:h||o++,prefix:g||"",delimiter:E,optional:x,repeat:_,partial:w,asterisk:!!k,pattern:S?c(S):k?".*":"[^"+s(E)+"]+?"})}}return a<e.length&&(l+=e.substr(a)),l&&i.push(l),i}function a(e){return encodeURI(e).replace(/[\/?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()}))}function l(e,n){for(var t=new Array(e.length),r=0;r<e.length;r++)"object"==typeof e[r]&&(t[r]=new RegExp("^(?:"+e[r].pattern+")$",p(n)));return function(n,r){for(var o="",l=n||{},s=(r||{}).pretty?a:encodeURIComponent,c=0;c<e.length;c++){var u=e[c];if("string"!=typeof u){var p,d=l[u.name];if(null==d){if(u.optional){u.partial&&(o+=u.prefix);continue}throw new TypeError('Expected "'+u.name+'" to be defined')}if(i(d)){if(!u.repeat)throw new TypeError('Expected "'+u.name+'" to not repeat, but received `'+JSON.stringify(d)+"`");if(0===d.length){if(u.optional)continue;throw new TypeError('Expected "'+u.name+'" to not be empty')}for(var f=0;f<d.length;f++){if(p=s(d[f]),!t[c].test(p))throw new TypeError('Expected all "'+u.name+'" to match "'+u.pattern+'", but received `'+JSON.stringify(p)+"`");o+=(0===f?u.prefix:u.delimiter)+p}}else{if(p=u.asterisk?encodeURI(d).replace(/[?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()})):s(d),!t[c].test(p))throw new TypeError('Expected "'+u.name+'" to match "'+u.pattern+'", but received "'+p+'"');o+=u.prefix+p}}else o+=u}return o}}function s(e){return e.replace(/([.+*?=^!:${}()[\]|\/\\])/g,"\\$1")}function c(e){return e.replace(/([=!:$\/()])/g,"\\$1")}function u(e,n){return e.keys=n,e}function p(e){return e&&e.sensitive?"":"i"}function d(e,n,t){i(n)||(t=n||t,n=[]);for(var r=(t=t||{}).strict,o=!1!==t.end,a="",l=0;l<e.length;l++){var c=e[l];if("string"==typeof c)a+=s(c);else{var d=s(c.prefix),f="(?:"+c.pattern+")";n.push(c),c.repeat&&(f+="(?:"+d+f+")*"),a+=f=c.optional?c.partial?d+"("+f+")?":"(?:"+d+"("+f+"))?":d+"("+f+")"}}var m=s(t.delimiter||"/"),g=a.slice(-m.length)===m;return r||(a=(g?a.slice(0,-m.length):a)+"(?:"+m+"(?=$))?"),a+=o?"$":r&&g?"":"(?="+m+"|$)",u(new RegExp("^"+a,p(t)),n)}function f(e,n,t){return i(n)||(t=n||t,n=[]),t=t||{},e instanceof RegExp?function(e,n){var t=e.source.match(/\((?!\?)/g);if(t)for(var i=0;i<t.length;i++)n.push({name:i,prefix:null,delimiter:null,optional:!1,repeat:!1,partial:!1,asterisk:!1,pattern:null});return u(e,n)}(e,n):i(e)?function(e,n,t){for(var i=[],r=0;r<e.length;r++)i.push(f(e[r],n,t).source);return u(new RegExp("(?:"+i.join("|")+")",p(t)),n)}(e,n,t):function(e,n,t){return d(o(e,t),n,t)}(e,n,t)}},function(e,n,t){"use strict";t.r(n);var i=t(10);t.d(n,"MemoryRouter",(function(){return i.d})),t.d(n,"Prompt",(function(){return i.f})),t.d(n,"Redirect",(function(){return i.g})),t.d(n,"Route",(function(){return i.h})),t.d(n,"Router",(function(){return i.i})),t.d(n,"StaticRouter",(function(){return i.j})),t.d(n,"Switch",(function(){return i.k})),t.d(n,"generatePath",(function(){return i.l})),t.d(n,"matchPath",(function(){return i.m})),t.d(n,"useHistory",(function(){return i.n})),t.d(n,"useLocation",(function(){return i.o})),t.d(n,"useParams",(function(){return i.p})),t.d(n,"useRouteMatch",(function(){return i.q})),t.d(n,"withRouter",(function(){return i.r})),t.d(n,"BrowserRouter",(function(){return i.a})),t.d(n,"HashRouter",(function(){return i.b})),t.d(n,"Link",(function(){return i.c})),t.d(n,"NavLink",(function(){return i.e}))},function(e,n,t){"use strict";var i,r,o,a=(i=0,r={util:{encode:function(e){return e instanceof o?new o(e.type,r.util.encode(e.content),e.alias):"Array"===r.util.type(e)?e.map(r.util.encode):e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).match(/\[object (\w+)\]/)[1]},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++i}),e.__id},clone:function(e,n){var t=r.util.type(e);switch(n=n||{},t){case"Object":if(n[r.util.objId(e)])return n[r.util.objId(e)];var i={};for(var o in n[r.util.objId(e)]=i,e)e.hasOwnProperty(o)&&(i[o]=r.util.clone(e[o],n));return i;case"Array":return n[r.util.objId(e)]?n[r.util.objId(e)]:(i=[],n[r.util.objId(e)]=i,e.forEach((function(e,t){i[t]=r.util.clone(e,n)})),i)}return e}},languages:{extend:function(e,n){var t=r.util.clone(r.languages[e]);for(var i in n)t[i]=n[i];return t},insertBefore:function(e,n,t,i){var o=(i=i||r.languages)[e];if(2==arguments.length){for(var a in t=arguments[1])t.hasOwnProperty(a)&&(o[a]=t[a]);return o}var l={};for(var s in o)if(o.hasOwnProperty(s)){if(s==n)for(var a in t)t.hasOwnProperty(a)&&(l[a]=t[a]);l[s]=o[s]}return r.languages.DFS(r.languages,(function(n,t){t===i[e]&&n!=e&&(this[n]=l)})),i[e]=l},DFS:function(e,n,t,i){for(var o in i=i||{},e)e.hasOwnProperty(o)&&(n.call(e,o,e[o],t||o),"Object"!==r.util.type(e[o])||i[r.util.objId(e[o])]?"Array"!==r.util.type(e[o])||i[r.util.objId(e[o])]||(i[r.util.objId(e[o])]=!0,r.languages.DFS(e[o],n,o,i)):(i[r.util.objId(e[o])]=!0,r.languages.DFS(e[o],n,null,i)))}},plugins:{},highlight:function(e,n,t){var i={code:e,grammar:n,language:t};return r.hooks.run("before-tokenize",i),i.tokens=r.tokenize(i.code,i.grammar),r.hooks.run("after-tokenize",i),o.stringify(r.util.encode(i.tokens),i.language)},matchGrammar:function(e,n,t,i,o,a,l){var s=r.Token;for(var c in t)if(t.hasOwnProperty(c)&&t[c]){if(c==l)return;var u=t[c];u="Array"===r.util.type(u)?u:[u];for(var p=0;p<u.length;++p){var d=u[p],f=d.inside,m=!!d.lookbehind,g=!!d.greedy,h=0,v=d.alias;if(g&&!d.pattern.global){var b=d.pattern.toString().match(/[imuy]*$/)[0];d.pattern=RegExp(d.pattern.source,b+"g")}d=d.pattern||d;for(var y=i,k=o;y<n.length;k+=n[y].length,++y){var w=n[y];if(n.length>e.length)return;if(!(w instanceof s)){if(g&&y!=n.length-1){if(d.lastIndex=k,!(P=d.exec(e)))break;for(var _=P.index+(m?P[1].length:0),x=P.index+P[0].length,E=y,S=k,T=n.length;E<T&&(S<x||!n[E].type&&!n[E-1].greedy);++E)_>=(S+=n[E].length)&&(++y,k=S);if(n[y]instanceof s)continue;C=E-y,w=e.slice(k,S),P.index-=k}else{d.lastIndex=0;var P=d.exec(w),C=1}if(P){m&&(h=P[1]?P[1].length:0),x=(_=P.index+h)+(P=P[0].slice(h)).length;var O=w.slice(0,_),A=w.slice(x),R=[y,C];O&&(++y,k+=O.length,R.push(O));var N=new s(c,f?r.tokenize(P,f):P,v,P,g);if(R.push(N),A&&R.push(A),Array.prototype.splice.apply(n,R),1!=C&&r.matchGrammar(e,n,t,y,k,!0,c),a)break}else if(a)break}}}}},hooks:{add:function(){},run:function(e,n){}},tokenize:function(e,n,t){var i=[e],o=n.rest;if(o){for(var a in o)n[a]=o[a];delete n.rest}return r.matchGrammar(e,i,n,0,0,!1),i}},(o=r.Token=function(e,n,t,i,r){this.type=e,this.content=n,this.alias=t,this.length=0|(i||"").length,this.greedy=!!r}).stringify=function(e,n,t){if("string"==typeof e)return e;if("Array"===r.util.type(e))return e.map((function(t){return o.stringify(t,n,e)})).join("");var i={type:e.type,content:o.stringify(e.content,n,t),tag:"span",classes:["token",e.type],attributes:{},language:n,parent:t};if(e.alias){var a="Array"===r.util.type(e.alias)?e.alias:[e.alias];Array.prototype.push.apply(i.classes,a)}var l=Object.keys(i.attributes).map((function(e){return e+'="'+(i.attributes[e]||"").replace(/"/g,"&quot;")+'"'})).join(" ");return"<"+i.tag+' class="'+i.classes.join(" ")+'"'+(l?" "+l:"")+">"+i.content+"</"+i.tag+">"},r);a.languages.markup={comment:/<!--[\s\S]*?-->/,prolog:/<\?[\s\S]+?\?>/,doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|<!--(?:[^-]|-(?!->))*-->)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^<!|>$|[[\]]/,"doctype-tag":/^DOCTYPE/,name:/[^\s<>'"]+/}},cdata:/<!\[CDATA\[[\s\S]*?]]>/i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},a.languages.markup.tag.inside["attr-value"].inside.entity=a.languages.markup.entity,a.languages.markup.doctype.inside["internal-subset"].inside=a.languages.markup,a.hooks.add("wrap",(function(e){"entity"===e.type&&(e.attributes.title=e.content.replace(/&amp;/,"&"))})),Object.defineProperty(a.languages.markup.tag,"addInlined",{value:function(e,n){var t={};t["language-"+n]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:a.languages[n]},t.cdata=/^<!\[CDATA\[|\]\]>$/i;var i={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:t}};i["language-"+n]={pattern:/[\s\S]+/,inside:a.languages[n]};var r={};r[e]={pattern:RegExp(/(<__[^>]*>)(?:<!\[CDATA\[(?:[^\]]|\](?!\]>))*\]\]>|(?!<!\[CDATA\[)[\s\S])*?(?=<\/__>)/.source.replace(/__/g,(function(){return e})),"i"),lookbehind:!0,greedy:!0,inside:i},a.languages.insertBefore("markup","cdata",r)}}),a.languages.html=a.languages.markup,a.languages.mathml=a.languages.markup,a.languages.svg=a.languages.markup,a.languages.xml=a.languages.extend("markup",{}),a.languages.ssml=a.languages.xml,a.languages.atom=a.languages.xml,a.languages.rss=a.languages.xml,function(e){var n="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",t={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},i={bash:t,environment:{pattern:RegExp("\\$"+n),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/=?|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+n),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|x[0-9a-fA-F]{1,2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)\w+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b\w+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+n),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+?)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:i},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:t}},{pattern:/(^|[^\\](?:\\\\)*)(["'])(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|(?!\2)[^\\`$])*\2/,lookbehind:!0,greedy:!0,inside:i}],environment:{pattern:RegExp("\\$?"+n),alias:"constant"},variable:i.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|aptitude|apt-cache|apt-get|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:if|then|else|elif|fi|for|while|in|case|esac|function|select|do|done|until)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|break|cd|continue|eval|exec|exit|export|getopts|hash|pwd|readonly|return|shift|test|times|trap|umask|unset|alias|bind|builtin|caller|command|declare|echo|enable|help|let|local|logout|mapfile|printf|read|readarray|source|type|typeset|ulimit|unalias|set|shopt)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:true|false)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|==?|!=?|=~|<<[<-]?|[&\d]?>>|\d?[<>]&?|&[>&]?|\|[&|]?|<=?|>=?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},t.inside=e.languages.bash;for(var r=["comment","function-name","for-or-select","assign-left","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],o=i.variable[1].inside,a=0;a<r.length;a++)o[r[a]]=e.languages.bash[r[a]];e.languages.shell=e.languages.bash}(a),a.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|interface|extends|implements|trait|instanceof|new)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue)\b/,boolean:/\b(?:true|false)\b/,function:/\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/},a.languages.c=a.languages.extend("clike",{comment:{pattern:/\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},"class-name":{pattern:/(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/,lookbehind:!0},keyword:/\b(?:__attribute__|_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while)\b/,function:/[a-z_]\w*(?=\s*\()/i,number:/(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i,operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/}),a.languages.insertBefore("c","string",{macro:{pattern:/(^\s*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im,lookbehind:!0,greedy:!0,alias:"property",inside:{string:[{pattern:/^(#\s*include\s*)<[^>]+>/,lookbehind:!0},a.languages.c.string],comment:a.languages.c.comment,"macro-name":[{pattern:/(^#\s*define\s+)\w+\b(?!\()/i,lookbehind:!0},{pattern:/(^#\s*define\s+)\w+\b(?=\()/i,lookbehind:!0,alias:"function"}],directive:{pattern:/^(#\s*)[a-z]+/,lookbehind:!0,alias:"keyword"},"directive-hash":/^#/,punctuation:/##|\\(?=[\r\n])/,expression:{pattern:/\S[\s\S]*/,inside:a.languages.c}}},constant:/\b(?:__FILE__|__LINE__|__DATE__|__TIME__|__TIMESTAMP__|__func__|EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|stdin|stdout|stderr)\b/}),delete a.languages.c.boolean,function(e){var n=/\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char8_t|char16_t|char32_t|class|compl|concept|const|consteval|constexpr|constinit|const_cast|continue|co_await|co_return|co_yield|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|float|for|friend|goto|if|inline|int|int8_t|int16_t|int32_t|int64_t|uint8_t|uint16_t|uint32_t|uint64_t|long|mutable|namespace|new|noexcept|nullptr|operator|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/;e.languages.cpp=e.languages.extend("c",{"class-name":[{pattern:RegExp(/(\b(?:class|concept|enum|struct|typename)\s+)(?!<keyword>)\w+/.source.replace(/<keyword>/g,(function(){return n.source}))),lookbehind:!0},/\b[A-Z]\w*(?=\s*::\s*\w+\s*\()/,/\b[A-Z_]\w*(?=\s*::\s*~\w+\s*\()/i,/\w+(?=\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*::\s*\w+\s*\()/],keyword:n,number:{pattern:/(?:\b0b[01']+|\b0x(?:[\da-f']+(?:\.[\da-f']*)?|\.[\da-f']+)(?:p[+-]?[\d']+)?|(?:\b[\d']+(?:\.[\d']*)?|\B\.[\d']+)(?:e[+-]?[\d']+)?)[ful]{0,4}/i,greedy:!0},operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/,boolean:/\b(?:true|false)\b/}),e.languages.insertBefore("cpp","string",{"raw-string":{pattern:/R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/,alias:"string",greedy:!0}}),e.languages.insertBefore("cpp","class-name",{"base-clause":{pattern:/(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/,lookbehind:!0,greedy:!0,inside:e.languages.extend("cpp",{})}}),e.languages.insertBefore("inside","operator",{"class-name":/\b[a-z_]\w*\b(?!\s*::)/i},e.languages.cpp["base-clause"])}(a),function(e){var n=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-](?:[^;{\s]|\s+(?![\s{]))*(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+n.source+"|"+/(?:[^\\\r\n()"']|\\[\s\S])*/.source+")\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+n.source+"$"),alias:"url"}}},selector:RegExp("[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+n.source+")*(?=\\s*\\{)"),string:{pattern:n,greedy:!0},property:/(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,important:/!important\b/i,function:/[-a-z0-9]+(?=\()/i,punctuation:/[(){};:,]/},e.languages.css.atrule.inside.rest=e.languages.css;var t=e.languages.markup;t&&(t.tag.addInlined("style","css"),e.languages.insertBefore("inside","attr-value",{"style-attr":{pattern:/(^|["'\s])style\s*=\s*(?:"[^"]*"|'[^']*')/i,lookbehind:!0,inside:{"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{style:{pattern:/(["'])[\s\S]+(?=["']$)/,lookbehind:!0,alias:"language-css",inside:e.languages.css},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},"attr-name":/^style/i}}},t.tag))}(a),function(e){var n,t=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css.selector={pattern:e.languages.css.selector,inside:n={"pseudo-element":/:(?:after|before|first-letter|first-line|selection)|::[-\w]+/,"pseudo-class":/:[-\w]+/,class:/\.[-\w]+/,id:/#[-\w]+/,attribute:{pattern:RegExp("\\[(?:[^[\\]\"']|"+t.source+")*\\]"),greedy:!0,inside:{punctuation:/^\[|\]$/,"case-sensitivity":{pattern:/(\s)[si]$/i,lookbehind:!0,alias:"keyword"},namespace:{pattern:/^(\s*)(?:(?!\s)[-*\w\xA0-\uFFFF])*\|(?!=)/,lookbehind:!0,inside:{punctuation:/\|$/}},"attr-name":{pattern:/^(\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+/,lookbehind:!0},"attr-value":[t,{pattern:/(=\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+(?=\s*$)/,lookbehind:!0}],operator:/[|~*^$]?=/}},"n-th":[{pattern:/(\(\s*)[+-]?\d*[\dn](?:\s*[+-]\s*\d+)?(?=\s*\))/,lookbehind:!0,inside:{number:/[\dn]+/,operator:/[+-]/}},{pattern:/(\(\s*)(?:even|odd)(?=\s*\))/i,lookbehind:!0}],combinator:/>|\+|~|\|\|/,punctuation:/[(),]/}},e.languages.css.atrule.inside["selector-function-argument"].inside=n,e.languages.insertBefore("css","property",{variable:{pattern:/(^|[^-\w\xA0-\uFFFF])--(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*/i,lookbehind:!0}});var i={pattern:/(\b\d+)(?:%|[a-z]+\b)/,lookbehind:!0},r={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0};e.languages.insertBefore("css","function",{operator:{pattern:/(\s)[+\-*\/](?=\s)/,lookbehind:!0},hexcode:{pattern:/\B#(?:[\da-f]{1,2}){3,4}\b/i,alias:"color"},color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:rgb|hsl)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:rgb|hsl)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:i,number:r,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:i,number:r})}(a),a.languages.javascript=a.languages.extend("clike",{"class-name":[a.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:prototype|constructor))/,lookbehind:!0}],keyword:[{pattern:/((?:^|})\s*)(?:catch|finally)\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|for|from|function|(?:get|set)(?=\s*[\[$\w\xA0-\uFFFF])|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:/\b(?:(?:0[xX](?:[\dA-Fa-f](?:_[\dA-Fa-f])?)+|0[bB](?:[01](?:_[01])?)+|0[oO](?:[0-7](?:_[0-7])?)+)n?|(?:\d(?:_\d)?)+n|NaN|Infinity)\b|(?:\b(?:\d(?:_\d)?)+\.?(?:\d(?:_\d)?)*|\B\.(?:\d(?:_\d)?)+)(?:[Ee][+-]?(?:\d(?:_\d)?)+)?/,operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),a.languages.javascript["class-name"][0].pattern=/(\b(?:class|interface|extends|implements|instanceof|new)\s+)[\w.\\]+/,a.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*]|\\.|[^/\\\[\r\n])+\/[gimyus]{0,6}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:a.languages.regex},"regex-flags":/[a-z]+$/,"regex-delimiter":/^\/|\/$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:a.languages.javascript},{pattern:/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,inside:a.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:a.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:a.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),a.languages.insertBefore("javascript","string",{"template-string":{pattern:/`(?:\\[\s\S]|\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}|(?!\${)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\${|}$/,alias:"punctuation"},rest:a.languages.javascript}},string:/[\s\S]+/}}}),a.languages.markup&&a.languages.markup.tag.addInlined("script","javascript"),a.languages.js=a.languages.javascript,function(e){var n=e.util.clone(e.languages.javascript);e.languages.jsx=e.languages.extend("markup",n),e.languages.jsx.tag.pattern=/<\/?(?:[\w.:-]+(?:\s+(?:[\w.:$-]+(?:=(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s{'">=]+|\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}))?|\{\s*\.{3}\s*[a-z_$][\w$]*(?:\.[a-z_$][\w$]*)*\s*\}))*\s*\/?)?>/i,e.languages.jsx.tag.inside.tag.pattern=/^<\/?[^\s>\/]*/i,e.languages.jsx.tag.inside["attr-value"].pattern=/=(?!\{)(?:"(?:\\[^]|[^\\"])*"|'(?:\\[^]|[^\\'])*'|[^\s'">]+)/i,e.languages.jsx.tag.inside.tag.inside["class-name"]=/^[A-Z]\w*(?:\.[A-Z]\w*)*$/,e.languages.insertBefore("inside","attr-name",{spread:{pattern:/\{\s*\.{3}\s*[a-z_$][\w$]*(?:\.[a-z_$][\w$]*)*\s*\}/,inside:{punctuation:/\.{3}|[{}.]/,"attr-value":/\w+/}}},e.languages.jsx.tag),e.languages.insertBefore("inside","attr-value",{script:{pattern:/=(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\})/i,inside:{"script-punctuation":{pattern:/^=(?={)/,alias:"punctuation"},rest:e.languages.jsx},alias:"language-javascript"}},e.languages.jsx.tag);var t=function(e){return e?"string"==typeof e?e:"string"==typeof e.content?e.content:e.content.map(t).join(""):""},i=function(n){for(var r=[],o=0;o<n.length;o++){var a=n[o],l=!1;if("string"!=typeof a&&("tag"===a.type&&a.content[0]&&"tag"===a.content[0].type?"</"===a.content[0].content[0].content?r.length>0&&r[r.length-1].tagName===t(a.content[0].content[1])&&r.pop():"/>"===a.content[a.content.length-1].content||r.push({tagName:t(a.content[0].content[1]),openedBraces:0}):r.length>0&&"punctuation"===a.type&&"{"===a.content?r[r.length-1].openedBraces++:r.length>0&&r[r.length-1].openedBraces>0&&"punctuation"===a.type&&"}"===a.content?r[r.length-1].openedBraces--:l=!0),(l||"string"==typeof a)&&r.length>0&&0===r[r.length-1].openedBraces){var s=t(a);o<n.length-1&&("string"==typeof n[o+1]||"plain-text"===n[o+1].type)&&(s+=t(n[o+1]),n.splice(o+1,1)),o>0&&("string"==typeof n[o-1]||"plain-text"===n[o-1].type)&&(s=t(n[o-1])+s,n.splice(o-1,1),o--),n[o]=new e.Token("plain-text",s,null,s)}a.content&&"string"!=typeof a.content&&i(a.content)}};e.hooks.add("after-tokenize",(function(e){"jsx"!==e.language&&"tsx"!==e.language||i(e.tokens)}))}(a),function(e){function n(e,n){return RegExp(e.replace(/<ID>/g,(function(){return/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/.source})),n)}e.languages.insertBefore("javascript","function-variable",{"method-variable":{pattern:RegExp("(\\.\\s*)"+e.languages.javascript["function-variable"].pattern.source),lookbehind:!0,alias:["function-variable","method","function","property-access"]}}),e.languages.insertBefore("javascript","function",{method:{pattern:RegExp("(\\.\\s*)"+e.languages.javascript.function.source),lookbehind:!0,alias:["function","property-access"]}}),e.languages.insertBefore("javascript","constant",{"known-class-name":[{pattern:/\b(?:(?:(?:Uint|Int)(?:8|16|32)|Uint8Clamped|Float(?:32|64))?Array|ArrayBuffer|BigInt|Boolean|DataView|Date|Error|Function|Intl|JSON|Math|Number|Object|Promise|Proxy|Reflect|RegExp|String|Symbol|(?:Weak)?(?:Set|Map)|WebAssembly)\b/,alias:"class-name"},{pattern:/\b(?:[A-Z]\w*)Error\b/,alias:"class-name"}]}),e.languages.insertBefore("javascript","keyword",{imports:{pattern:n(/(\bimport\b\s*)(?:<ID>(?:\s*,\s*(?:\*\s*as\s+<ID>|\{[^{}]*\}))?|\*\s*as\s+<ID>|\{[^{}]*\})(?=\s*\bfrom\b)/.source),lookbehind:!0,inside:e.languages.javascript},exports:{pattern:n(/(\bexport\b\s*)(?:\*(?:\s*as\s+<ID>)?(?=\s*\bfrom\b)|\{[^{}]*\})/.source),lookbehind:!0,inside:e.languages.javascript}}),e.languages.javascript.keyword.unshift({pattern:/\b(?:as|default|export|from|import)\b/,alias:"module"},{pattern:/\b(?:await|break|catch|continue|do|else|for|finally|if|return|switch|throw|try|while|yield)\b/,alias:"control-flow"},{pattern:/\bnull\b/,alias:["null","nil"]},{pattern:/\bundefined\b/,alias:"nil"}),e.languages.insertBefore("javascript","operator",{spread:{pattern:/\.{3}/,alias:"operator"},arrow:{pattern:/=>/,alias:"operator"}}),e.languages.insertBefore("javascript","punctuation",{"property-access":{pattern:n(/(\.\s*)#?<ID>/.source),lookbehind:!0},"maybe-class-name":{pattern:/(^|[^$\w\xA0-\uFFFF])[A-Z][$\w\xA0-\uFFFF]+/,lookbehind:!0},dom:{pattern:/\b(?:document|location|navigator|performance|(?:local|session)Storage|window)\b/,alias:"variable"},console:{pattern:/\bconsole(?=\s*\.)/,alias:"class-name"}});for(var t=["function","function-variable","method","method-variable","property-access"],i=0;i<t.length;i++){var r=t[i],o=e.languages.javascript[r];"RegExp"===e.util.type(o)&&(o=e.languages.javascript[r]={pattern:o});var a=o.inside||{};o.inside=a,a["maybe-class-name"]=/^[A-Z][\s\S]*/}}(a),function(e){var n=/#(?!\{).+/,t={pattern:/#\{[^}]+\}/,alias:"variable"};e.languages.coffeescript=e.languages.extend("javascript",{comment:n,string:[{pattern:/'(?:\\[\s\S]|[^\\'])*'/,greedy:!0},{pattern:/"(?:\\[\s\S]|[^\\"])*"/,greedy:!0,inside:{interpolation:t}}],keyword:/\b(?:and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield)\b/,"class-member":{pattern:/@(?!\d)\w+/,alias:"variable"}}),e.languages.insertBefore("coffeescript","comment",{"multiline-comment":{pattern:/###[\s\S]+?###/,alias:"comment"},"block-regex":{pattern:/\/{3}[\s\S]*?\/{3}/,alias:"regex",inside:{comment:n,interpolation:t}}}),e.languages.insertBefore("coffeescript","string",{"inline-javascript":{pattern:/`(?:\\[\s\S]|[^\\`])*`/,inside:{delimiter:{pattern:/^`|`$/,alias:"punctuation"},script:{pattern:/[\s\S]+/,alias:"language-javascript",inside:e.languages.javascript}}},"multiline-string":[{pattern:/'''[\s\S]*?'''/,greedy:!0,alias:"string"},{pattern:/"""[\s\S]*?"""/,greedy:!0,alias:"string",inside:{interpolation:t}}]}),e.languages.insertBefore("coffeescript","keyword",{property:/(?!\d)\w+(?=\s*:(?!:))/}),delete e.languages.coffeescript["template-string"],e.languages.coffee=e.languages.coffeescript}(a),function(e){e.languages.diff={coord:[/^(?:\*{3}|-{3}|\+{3}).*$/m,/^@@.*@@$/m,/^\d.*$/m]};var n={"deleted-sign":"-","deleted-arrow":"<","inserted-sign":"+","inserted-arrow":">",unchanged:" ",diff:"!"};Object.keys(n).forEach((function(t){var i=n[t],r=[];/^\w+$/.test(t)||r.push(/\w+/.exec(t)[0]),"diff"===t&&r.push("bold"),e.languages.diff[t]={pattern:RegExp("^(?:["+i+"].*(?:\r\n?|\n|(?![\\s\\S])))+","m"),alias:r,inside:{line:{pattern:/(.)(?=[\s\S]).*(?:\r\n?|\n)?/,lookbehind:!0},prefix:{pattern:/[\s\S]/,alias:/\w+/.exec(t)[0]}}}})),Object.defineProperty(e.languages.diff,"PREFIXES",{value:n})}(a),a.languages.git={comment:/^#.*/m,deleted:/^[-\u2013].*/m,inserted:/^\+.*/m,string:/("|')(?:\\.|(?!\1)[^\\\r\n])*\1/m,command:{pattern:/^.*\$ git .*$/m,inside:{parameter:/\s--?\w+/m}},coord:/^@@.*@@$/m,"commit-sha1":/^commit \w{40}$/m},a.languages.go=a.languages.extend("clike",{string:{pattern:/(["'`])(?:\\[\s\S]|(?!\1)[^\\])*\1/,greedy:!0},keyword:/\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,boolean:/\b(?:_|iota|nil|true|false)\b/,number:/(?:\b0x[a-f\d]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[-+]?\d+)?)i?/i,operator:/[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,builtin:/\b(?:bool|byte|complex(?:64|128)|error|float(?:32|64)|rune|string|u?int(?:8|16|32|64)?|uintptr|append|cap|close|complex|copy|delete|imag|len|make|new|panic|print(?:ln)?|real|recover)\b/}),delete a.languages.go["class-name"],a.languages.graphql={comment:/#.*/,description:{pattern:/(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i,greedy:!0,alias:"string",inside:{"language-markdown":{pattern:/(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/,lookbehind:!0,inside:a.languages.markdown}}},string:{pattern:/"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/,greedy:!0},number:/(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,boolean:/\b(?:true|false)\b/,variable:/\$[a-z_]\w*/i,directive:{pattern:/@[a-z_]\w*/i,alias:"function"},"attr-name":{pattern:/[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i,greedy:!0},"class-name":{pattern:/(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*)[a-zA-Z_]\w*/,lookbehind:!0},fragment:{pattern:/(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},keyword:/\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/,operator:/[!=|&]|\.{3}/,punctuation:/[!(){}\[\]:=,]/,constant:/\b(?!ID\b)[A-Z][A-Z_\d]*\b/},function(e){function n(e,n){return"___"+e.toUpperCase()+n+"___"}Object.defineProperties(e.languages["markup-templating"]={},{buildPlaceholders:{value:function(t,i,r,o){if(t.language===i){var a=t.tokenStack=[];t.code=t.code.replace(r,(function(e){if("function"==typeof o&&!o(e))return e;for(var r,l=a.length;-1!==t.code.indexOf(r=n(i,l));)++l;return a[l]=e,r})),t.grammar=e.languages.markup}}},tokenizePlaceholders:{value:function(t,i){if(t.language===i&&t.tokenStack){t.grammar=e.languages[i];var r=0,o=Object.keys(t.tokenStack);!function a(l){for(var s=0;s<l.length&&!(r>=o.length);s++){var c=l[s];if("string"==typeof c||c.content&&"string"==typeof c.content){var u=o[r],p=t.tokenStack[u],d="string"==typeof c?c:c.content,f=n(i,u),m=d.indexOf(f);if(m>-1){++r;var g=d.substring(0,m),h=new e.Token(i,e.tokenize(p,t.grammar),"language-"+i,p),v=d.substring(m+f.length),b=[];g&&b.push.apply(b,a([g])),b.push(h),v&&b.push.apply(b,a([v])),"string"==typeof c?l.splice.apply(l,[s,1].concat(b)):c.content=b}}else c.content&&a(c.content)}return l}(t.tokens)}}}})}(a),function(e){e.languages.handlebars={comment:/\{\{![\s\S]*?\}\}/,delimiter:{pattern:/^\{\{\{?|\}\}\}?$/i,alias:"punctuation"},string:/(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][+-]?\d+)?/,boolean:/\b(?:true|false)\b/,block:{pattern:/^(\s*(?:~\s*)?)[#\/]\S+?(?=\s*(?:~\s*)?$|\s)/i,lookbehind:!0,alias:"keyword"},brackets:{pattern:/\[[^\]]+\]/,inside:{punctuation:/\[|\]/,variable:/[\s\S]+/}},punctuation:/[!"#%&':()*+,.\/;<=>@\[\\\]^`{|}~]/,variable:/[^!"#%&'()*+,\/;<=>@\[\\\]^`{|}~\s]+/},e.hooks.add("before-tokenize",(function(n){e.languages["markup-templating"].buildPlaceholders(n,"handlebars",/\{\{\{[\s\S]+?\}\}\}|\{\{[\s\S]+?\}\}/g)})),e.hooks.add("after-tokenize",(function(n){e.languages["markup-templating"].tokenizePlaceholders(n,"handlebars")}))}(a),a.languages.json={property:{pattern:/"(?:\\.|[^\\"\r\n])*"(?=\s*:)/,greedy:!0},string:{pattern:/"(?:\\.|[^\\"\r\n])*"(?!\s*:)/,greedy:!0},comment:{pattern:/\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},number:/-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,punctuation:/[{}[\],]/,operator:/:/,boolean:/\b(?:true|false)\b/,null:{pattern:/\bnull\b/,alias:"keyword"}},a.languages.webmanifest=a.languages.json,a.languages.less=a.languages.extend("css",{comment:[/\/\*[\s\S]*?\*\//,{pattern:/(^|[^\\])\/\/.*/,lookbehind:!0}],atrule:{pattern:/@[\w-](?:\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{punctuation:/[:()]/}},selector:{pattern:/(?:@\{[\w-]+\}|[^{};\s@])(?:@\{[\w-]+\}|\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};@\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{variable:/@+[\w-]+/}},property:/(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/i,operator:/[+\-*\/]/}),a.languages.insertBefore("less","property",{variable:[{pattern:/@[\w-]+\s*:/,inside:{punctuation:/:/}},/@@?[\w-]+/],"mixin-usage":{pattern:/([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/,lookbehind:!0,alias:"function"}}),a.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|[^\\\r\n])*/,lookbehind:!0},string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},builtin:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,symbol:{pattern:/^(?:[^:=\s]|[ \t]+(?![\s:]))+(?=\s*:(?!=))/m,inside:{variable:/\$+(?:(?!\$)[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:(?!\$)[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:[/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,{pattern:/(\()(?:addsuffix|abspath|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:s|list)?)(?=[ \t])/,lookbehind:!0}],operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/},function(e){var n=/(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?!\n|\r\n?))/.source;function t(e){return e=e.replace(/<inner>/g,(function(){return n})),RegExp(/((?:^|[^\\])(?:\\{2})*)/.source+"(?:"+e+")")}var i=/(?:\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\|\r\n`])+/.source,r=/\|?__(?:\|__)+\|?(?:(?:\n|\r\n?)|(?![\s\S]))/.source.replace(/__/g,(function(){return i})),o=/\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/.source;e.languages.markdown=e.languages.extend("markup",{}),e.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"font-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:e.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+r+o+"(?:"+r+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+r+o+")(?:"+r+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(i),inside:e.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+r+")"+o+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+r+"$"),inside:{"table-header":{pattern:RegExp(i),alias:"important",inside:e.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/``.+?``|`[^`\r\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:t(/\b__(?:(?!_)<inner>|_(?:(?!_)<inner>)+_)+__\b|\*\*(?:(?!\*)<inner>|\*(?:(?!\*)<inner>)+\*)+\*\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:t(/\b_(?:(?!_)<inner>|__(?:(?!_)<inner>)+__)+_\b|\*(?:(?!\*)<inner>|\*\*(?:(?!\*)<inner>)+\*\*)+\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:t(/(~~?)(?:(?!~)<inner>)+?\2/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:t(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(n){["url","bold","italic","strike"].forEach((function(t){n!==t&&(e.languages.markdown[n].inside.content.inside[t]=e.languages.markdown[t])}))})),e.hooks.add("after-tokenize",(function(e){"markdown"!==e.language&&"md"!==e.language||function e(n){if(n&&"string"!=typeof n)for(var t=0,i=n.length;t<i;t++){var r=n[t];if("code"===r.type){var o=r.content[1],a=r.content[3];if(o&&a&&"code-language"===o.type&&"code-block"===a.type&&"string"==typeof o.content){var l=o.content.replace(/\b#/g,"sharp").replace(/\b\+\+/g,"pp"),s="language-"+(l=(/[a-z][\w-]*/i.exec(l)||[""])[0].toLowerCase());a.alias?"string"==typeof a.alias?a.alias=[a.alias,s]:a.alias.push(s):a.alias=[s]}}else e(r.content)}}(e.tokens)})),e.hooks.add("wrap",(function(n){if("code-block"===n.type){for(var t="",i=0,r=n.classes.length;i<r;i++){var o=n.classes[i],a=/language-(.+)/.exec(o);if(a){t=a[1];break}}var l=e.languages[t];if(l){var s=n.content.replace(/&lt;/g,"<").replace(/&amp;/g,"&");n.content=e.highlight(s,l,t)}else if(t&&"none"!==t&&e.plugins.autoloader){var c="md-"+(new Date).valueOf()+"-"+Math.floor(1e16*Math.random());n.attributes.id=c,e.plugins.autoloader.loadLanguages(t,(function(){var n=document.getElementById(c);n&&(n.innerHTML=e.highlight(n.textContent,e.languages[t],t))}))}}})),e.languages.md=e.languages.markdown}(a),a.languages.objectivec=a.languages.extend("c",{string:/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1|@"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,keyword:/\b(?:asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|in|self|super)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/,operator:/-[->]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|\|?|[~^%?*\/@]/}),delete a.languages.objectivec["class-name"],a.languages.objc=a.languages.objectivec,a.languages.ocaml={comment:/\(\*[\s\S]*?\*\)/,string:[{pattern:/"(?:\\.|[^\\\r\n"])*"/,greedy:!0},{pattern:/(['`])(?:\\(?:\d+|x[\da-f]+|.)|(?!\1)[^\\\r\n])\1/i,greedy:!0}],number:/\b(?:0x[\da-f][\da-f_]+|(?:0[bo])?\d[\d_]*(?:\.[\d_]*)?(?:e[+-]?[\d_]+)?)/i,directive:{pattern:/\B#\w+/,alias:"important"},label:{pattern:/\B~\w+/,alias:"function"},"type-variable":{pattern:/\B'\w+/,alias:"function"},variant:{pattern:/`\w+/,alias:"variable"},module:{pattern:/\b[A-Z]\w+/,alias:"variable"},keyword:/\b(?:as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|module|mutable|new|nonrec|object|of|open|private|rec|sig|struct|then|to|try|type|val|value|virtual|when|where|while|with)\b/,boolean:/\b(?:false|true)\b/,operator:/:=|[=<>@^|&+\-*\/$%!?~][!$%&*+\-.\/:<=>?@^|~]*|\b(?:and|asr|land|lor|lsl|lsr|lxor|mod|or)\b/,punctuation:/[(){}\[\]|.,:;]|\b_\b/},a.languages.python={comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0},"string-interpolation":{pattern:/(?:f|rf|fr)(?:("""|''')[\s\S]*?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,greedy:!0,inside:{interpolation:{pattern:/((?:^|[^{])(?:{{)*){(?!{)(?:[^{}]|{(?!{)(?:[^{}]|{(?!{)(?:[^{}])+})+})+}/,lookbehind:!0,inside:{"format-spec":{pattern:/(:)[^:(){}]+(?=}$)/,lookbehind:!0},"conversion-option":{pattern:/![sra](?=[:}]$)/,alias:"punctuation"},rest:null}},string:/[\s\S]+/}},"triple-quoted-string":{pattern:/(?:[rub]|rb|br)?("""|''')[\s\S]*?\1/i,greedy:!0,alias:"string"},string:{pattern:/(?:[rub]|rb|br)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,greedy:!0},function:{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)\w+/i,lookbehind:!0},decorator:{pattern:/(^\s*)@\w+(?:\.\w+)*/im,lookbehind:!0,alias:["annotation","punctuation"],inside:{punctuation:/\./}},keyword:/\b(?:and|as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|print|raise|return|try|while|with|yield)\b/,builtin:/\b(?:__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip)\b/,boolean:/\b(?:True|False|None)\b/,number:/(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?j?\b/i,operator:/[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,punctuation:/[{}[\];(),.:]/},a.languages.python["string-interpolation"].inside.interpolation.inside.rest=a.languages.python,a.languages.py=a.languages.python,a.languages.reason=a.languages.extend("clike",{string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/,greedy:!0},"class-name":/\b[A-Z]\w*/,keyword:/\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/,operator:/\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:mod|land|lor|lxor|lsl|lsr|asr)\b/}),a.languages.insertBefore("reason","class-name",{character:{pattern:/'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/,alias:"string"},constructor:{pattern:/\b[A-Z]\w*\b(?!\s*\.)/,alias:"variable"},label:{pattern:/\b[a-z]\w*(?=::)/,alias:"symbol"}}),delete a.languages.reason.function,function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m,lookbehind:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,inside:{atrule:/(?:@[\w-]+|[+=])/m}}}),delete e.languages.sass.atrule;var n=/\$[-\w]+|#\{\$[-\w]+\}/,t=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|or|not)\b/,{pattern:/(\s+)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,inside:{punctuation:/:/,variable:n,operator:t}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:n,operator:t,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/,lookbehind:!0}})}(a),a.languages.scss=a.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-](?:\([^()]+\)|[^()\s]|\s+(?!\s))*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)?url(?=\()/i,selector:{pattern:/(?=\S)[^@;{}()]?(?:[^@;{}()\s]|\s+(?!\s)|#\{\$[-\w]+\})+(?=\s*\{(?:\}|\s|[^}][^:{}]*[:{][^}]+))/m,inside:{parent:{pattern:/&/,alias:"important"},placeholder:/%[-\w]+/,variable:/\$[-\w]+|#\{\$[-\w]+\}/}},property:{pattern:/(?:[-\w]|\$[-\w]|#\{\$[-\w]+\})+(?=\s*:)/,inside:{variable:/\$[-\w]+|#\{\$[-\w]+\}/}}}),a.languages.insertBefore("scss","atrule",{keyword:[/@(?:if|else(?: if)?|forward|for|each|while|import|use|extend|debug|warn|mixin|include|function|return|content)\b/i,{pattern:/( +)(?:from|through)(?= )/,lookbehind:!0}]}),a.languages.insertBefore("scss","important",{variable:/\$[-\w]+|#\{\$[-\w]+\}/}),a.languages.insertBefore("scss","function",{"module-modifier":{pattern:/\b(?:as|with|show|hide)\b/i,alias:"keyword"},placeholder:{pattern:/%[-\w]+/,alias:"selector"},statement:{pattern:/\B!(?:default|optional)\b/i,alias:"keyword"},boolean:/\b(?:true|false)\b/,null:{pattern:/\bnull\b/,alias:"keyword"},operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|or|not)(?=\s)/,lookbehind:!0}}),a.languages.scss.atrule.inside.rest=a.languages.scss,a.languages.sql={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,lookbehind:!0},variable:[{pattern:/@(["'`])(?:\\[\s\S]|(?!\1)[^\\])+\1/,greedy:!0},/@[\w.$]+/],string:{pattern:/(^|[^@\\])("|')(?:\\[\s\S]|(?!\2)[^\\]|\2\2)*\2/,greedy:!0,lookbehind:!0},function:/\b(?:AVG|COUNT|FIRST|FORMAT|LAST|LCASE|LEN|MAX|MID|MIN|MOD|NOW|ROUND|SUM|UCASE)(?=\s*\()/i,keyword:/\b(?:ACTION|ADD|AFTER|ALGORITHM|ALL|ALTER|ANALYZE|ANY|APPLY|AS|ASC|AUTHORIZATION|AUTO_INCREMENT|BACKUP|BDB|BEGIN|BERKELEYDB|BIGINT|BINARY|BIT|BLOB|BOOL|BOOLEAN|BREAK|BROWSE|BTREE|BULK|BY|CALL|CASCADED?|CASE|CHAIN|CHAR(?:ACTER|SET)?|CHECK(?:POINT)?|CLOSE|CLUSTERED|COALESCE|COLLATE|COLUMNS?|COMMENT|COMMIT(?:TED)?|COMPUTE|CONNECT|CONSISTENT|CONSTRAINT|CONTAINS(?:TABLE)?|CONTINUE|CONVERT|CREATE|CROSS|CURRENT(?:_DATE|_TIME|_TIMESTAMP|_USER)?|CURSOR|CYCLE|DATA(?:BASES?)?|DATE(?:TIME)?|DAY|DBCC|DEALLOCATE|DEC|DECIMAL|DECLARE|DEFAULT|DEFINER|DELAYED|DELETE|DELIMITERS?|DENY|DESC|DESCRIBE|DETERMINISTIC|DISABLE|DISCARD|DISK|DISTINCT|DISTINCTROW|DISTRIBUTED|DO|DOUBLE|DROP|DUMMY|DUMP(?:FILE)?|DUPLICATE|ELSE(?:IF)?|ENABLE|ENCLOSED|END|ENGINE|ENUM|ERRLVL|ERRORS|ESCAPED?|EXCEPT|EXEC(?:UTE)?|EXISTS|EXIT|EXPLAIN|EXTENDED|FETCH|FIELDS|FILE|FILLFACTOR|FIRST|FIXED|FLOAT|FOLLOWING|FOR(?: EACH ROW)?|FORCE|FOREIGN|FREETEXT(?:TABLE)?|FROM|FULL|FUNCTION|GEOMETRY(?:COLLECTION)?|GLOBAL|GOTO|GRANT|GROUP|HANDLER|HASH|HAVING|HOLDLOCK|HOUR|IDENTITY(?:_INSERT|COL)?|IF|IGNORE|IMPORT|INDEX|INFILE|INNER|INNODB|INOUT|INSERT|INT|INTEGER|INTERSECT|INTERVAL|INTO|INVOKER|ISOLATION|ITERATE|JOIN|KEYS?|KILL|LANGUAGE|LAST|LEAVE|LEFT|LEVEL|LIMIT|LINENO|LINES|LINESTRING|LOAD|LOCAL|LOCK|LONG(?:BLOB|TEXT)|LOOP|MATCH(?:ED)?|MEDIUM(?:BLOB|INT|TEXT)|MERGE|MIDDLEINT|MINUTE|MODE|MODIFIES|MODIFY|MONTH|MULTI(?:LINESTRING|POINT|POLYGON)|NATIONAL|NATURAL|NCHAR|NEXT|NO|NONCLUSTERED|NULLIF|NUMERIC|OFF?|OFFSETS?|ON|OPEN(?:DATASOURCE|QUERY|ROWSET)?|OPTIMIZE|OPTION(?:ALLY)?|ORDER|OUT(?:ER|FILE)?|OVER|PARTIAL|PARTITION|PERCENT|PIVOT|PLAN|POINT|POLYGON|PRECEDING|PRECISION|PREPARE|PREV|PRIMARY|PRINT|PRIVILEGES|PROC(?:EDURE)?|PUBLIC|PURGE|QUICK|RAISERROR|READS?|REAL|RECONFIGURE|REFERENCES|RELEASE|RENAME|REPEAT(?:ABLE)?|REPLACE|REPLICATION|REQUIRE|RESIGNAL|RESTORE|RESTRICT|RETURN(?:S|ING)?|REVOKE|RIGHT|ROLLBACK|ROUTINE|ROW(?:COUNT|GUIDCOL|S)?|RTREE|RULE|SAVE(?:POINT)?|SCHEMA|SECOND|SELECT|SERIAL(?:IZABLE)?|SESSION(?:_USER)?|SET(?:USER)?|SHARE|SHOW|SHUTDOWN|SIMPLE|SMALLINT|SNAPSHOT|SOME|SONAME|SQL|START(?:ING)?|STATISTICS|STATUS|STRIPED|SYSTEM_USER|TABLES?|TABLESPACE|TEMP(?:ORARY|TABLE)?|TERMINATED|TEXT(?:SIZE)?|THEN|TIME(?:STAMP)?|TINY(?:BLOB|INT|TEXT)|TOP?|TRAN(?:SACTIONS?)?|TRIGGER|TRUNCATE|TSEQUAL|TYPES?|UNBOUNDED|UNCOMMITTED|UNDEFINED|UNION|UNIQUE|UNLOCK|UNPIVOT|UNSIGNED|UPDATE(?:TEXT)?|USAGE|USE|USER|USING|VALUES?|VAR(?:BINARY|CHAR|CHARACTER|YING)|VIEW|WAITFOR|WARNINGS|WHEN|WHERE|WHILE|WITH(?: ROLLUP|IN)?|WORK|WRITE(?:TEXT)?|YEAR)\b/i,boolean:/\b(?:TRUE|FALSE|NULL)\b/i,number:/\b0x[\da-f]+\b|\b\d+(?:\.\d*)?|\B\.\d+\b/i,operator:/[-+*\/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?|\b(?:AND|BETWEEN|IN|LIKE|NOT|OR|IS|DIV|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b/i,punctuation:/[;[\]()`,.]/},function(e){var n={pattern:/(\b\d+)(?:%|[a-z]+)/,lookbehind:!0},t={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0},i={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},url:{pattern:/url\((["']?).*?\1\)/i,greedy:!0},string:{pattern:/("|')(?:(?!\1)[^\\\r\n]|\\(?:\r\n|[\s\S]))*\1/,greedy:!0},interpolation:null,func:null,important:/\B!(?:important|optional)\b/i,keyword:{pattern:/(^|\s+)(?:(?:if|else|for|return|unless)(?=\s+|$)|@[\w-]+)/,lookbehind:!0},hexcode:/#[\da-f]{3,6}/i,color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:rgb|hsl)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:rgb|hsl)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:n,number:t,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:n,boolean:/\b(?:true|false)\b/,operator:[/~|[+!\/%<>?=]=?|[-:]=|\*[*=]?|\.{2,3}|&&|\|\||\B-\B|\b(?:and|in|is(?: a| defined| not|nt)?|not|or)\b/],number:t,punctuation:/[{}()\[\];:,]/};i.interpolation={pattern:/\{[^\r\n}:]+\}/,alias:"variable",inside:{delimiter:{pattern:/^{|}$/,alias:"punctuation"},rest:i}},i.func={pattern:/[\w-]+\([^)]*\).*/,inside:{function:/^[^(]+/,rest:i}},e.languages.stylus={"atrule-declaration":{pattern:/(^\s*)@.+/m,lookbehind:!0,inside:{atrule:/^@[\w-]+/,rest:i}},"variable-declaration":{pattern:/(^[ \t]*)[\w$-]+\s*.?=[ \t]*(?:\{[^{}]*\}|\S.*|$)/m,lookbehind:!0,inside:{variable:/^\S+/,rest:i}},statement:{pattern:/(^[ \t]*)(?:if|else|for|return|unless)[ \t].+/m,lookbehind:!0,inside:{keyword:/^\S+/,rest:i}},"property-declaration":{pattern:/((?:^|\{)([ \t]*))(?:[\w-]|\{[^}\r\n]+\})+(?:\s*:\s*|[ \t]+)(?!\s)[^{\r\n]*(?:;|[^{\r\n,](?=$)(?!(?:\r?\n|\r)(?:\{|\2[ \t]+)))/m,lookbehind:!0,inside:{property:{pattern:/^[^\s:]+/,inside:{interpolation:i.interpolation}},rest:i}},selector:{pattern:/(^[ \t]*)(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)(?:(?:\r?\n|\r)(?:\1(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)))*(?:,$|\{|(?=(?:\r?\n|\r)(?:\{|\1[ \t]+)))/m,lookbehind:!0,inside:{interpolation:i.interpolation,comment:i.comment,punctuation:/[{},]/}},func:i.func,string:i.string,comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0,greedy:!0},interpolation:i.interpolation,punctuation:/[{}()\[\];:.]/}}(a),function(e){e.languages.typescript=e.languages.extend("javascript",{"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|type)\s+)(?!keyof\b)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?:\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},keyword:/\b(?:abstract|as|asserts|async|await|break|case|catch|class|const|constructor|continue|debugger|declare|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|is|keyof|let|module|namespace|new|null|of|package|private|protected|public|readonly|return|require|set|static|super|switch|this|throw|try|type|typeof|undefined|var|void|while|with|yield)\b/,builtin:/\b(?:string|Function|any|number|boolean|Array|symbol|console|Promise|unknown|never)\b/}),delete e.languages.typescript.parameter;var n=e.languages.extend("typescript",{});delete n["class-name"],e.languages.typescript["class-name"].inside=n,e.languages.insertBefore("typescript","function",{"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:n}}}}),e.languages.ts=e.languages.typescript}(a),function(e){var n=e.util.clone(e.languages.typescript);e.languages.tsx=e.languages.extend("jsx",n);var t=e.languages.tsx.tag;t.pattern=RegExp(/(^|[^\w$]|(?=<\/))/.source+"(?:"+t.pattern.source+")",t.pattern.flags),t.lookbehind=!0}(a),a.languages.wasm={comment:[/\(;[\s\S]*?;\)/,{pattern:/;;.*/,greedy:!0}],string:{pattern:/"(?:\\[\s\S]|[^"\\])*"/,greedy:!0},keyword:[{pattern:/\b(?:align|offset)=/,inside:{operator:/=/}},{pattern:/\b(?:(?:f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|nearest|neg?|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|store(?:8|16|32)?|sqrt|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))?|memory\.(?:grow|size))\b/,inside:{punctuation:/\./}},/\b(?:anyfunc|block|br(?:_if|_table)?|call(?:_indirect)?|data|drop|elem|else|end|export|func|get_(?:global|local)|global|if|import|local|loop|memory|module|mut|nop|offset|param|result|return|select|set_(?:global|local)|start|table|tee_local|then|type|unreachable)\b/],variable:/\$[\w!#$%&'*+\-./:<=>?@\\^_`|~]+/i,number:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/,punctuation:/[()]/},function(e){var n=/[*&][^\s[\]{},]+/,t=/!(?:<[\w\-%#;/?:@&=+$,.!~*'()[\]]+>|(?:[a-zA-Z\d-]*!)?[\w\-%#;/?:@&=+$.~*'()]+)?/,i="(?:"+t.source+"(?:[ \t]+"+n.source+")?|"+n.source+"(?:[ \t]+"+t.source+")?)",r=/(?:[^\s\x00-\x08\x0e-\x1f!"#%&'*,\-:>?@[\]`{|}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]|[?:-]<PLAIN>)(?:[ \t]*(?:(?![#:])<PLAIN>|:<PLAIN>))*/.source.replace(/<PLAIN>/g,(function(){return/[^\s\x00-\x08\x0e-\x1f,[\]{}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]/.source})),o=/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/.source;function a(e,n){n=(n||"").replace(/m/g,"")+"m";var t=/([:\-,[{]\s*(?:\s<<prop>>[ \t]+)?)(?:<<value>>)(?=[ \t]*(?:$|,|]|}|(?:[\r\n]\s*)?#))/.source.replace(/<<prop>>/g,(function(){return i})).replace(/<<value>>/g,(function(){return e}));return RegExp(t,n)}e.languages.yaml={scalar:{pattern:RegExp(/([\-:]\s*(?:\s<<prop>>[ \t]+)?[|>])[ \t]*(?:((?:\r?\n|\r)[ \t]+)\S[^\r\n]*(?:\2[^\r\n]+)*)/.source.replace(/<<prop>>/g,(function(){return i}))),lookbehind:!0,alias:"string"},comment:/#.*/,key:{pattern:RegExp(/((?:^|[:\-,[{\r\n?])[ \t]*(?:<<prop>>[ \t]+)?)<<key>>(?=\s*:\s)/.source.replace(/<<prop>>/g,(function(){return i})).replace(/<<key>>/g,(function(){return"(?:"+r+"|"+o+")"}))),lookbehind:!0,greedy:!0,alias:"atrule"},directive:{pattern:/(^[ \t]*)%.+/m,lookbehind:!0,alias:"important"},datetime:{pattern:a(/\d{4}-\d\d?-\d\d?(?:[tT]|[ \t]+)\d\d?:\d{2}:\d{2}(?:\.\d*)?(?:[ \t]*(?:Z|[-+]\d\d?(?::\d{2})?))?|\d{4}-\d{2}-\d{2}|\d\d?:\d{2}(?::\d{2}(?:\.\d*)?)?/.source),lookbehind:!0,alias:"number"},boolean:{pattern:a(/true|false/.source,"i"),lookbehind:!0,alias:"important"},null:{pattern:a(/null|~/.source,"i"),lookbehind:!0,alias:"important"},string:{pattern:a(o),lookbehind:!0,greedy:!0},number:{pattern:a(/[+-]?(?:0x[\da-f]+|0o[0-7]+|(?:\d+(?:\.\d*)?|\.?\d+)(?:e[+-]?\d+)?|\.inf|\.nan)/.source,"i"),lookbehind:!0},tag:t,important:n,punctuation:/---|[:[\]{}\-,|>?]|\.\.\./},e.languages.yml=e.languages.yaml}(a),n.a=a},function(e,n,t){"use strict";var i=t(0),r=t.n(i),o=t(35);n.a=function(e){return r.a.createElement(o.a,Object.assign({},e))}},function(e){e.exports=JSON.parse("{}")},function(e,n,t){"use strict";var i=t(9);n.a=i.b},function(e,n){var t;t=function(){return this}();try{t=t||new Function("return this")()}catch(i){"object"==typeof window&&(t=window)}e.exports=t},function(e,n,t){"use strict";e.exports=t(46)},function(e,n,t){"use strict";!function e(){if("undefined"!=typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(n){console.error(n)}}(),e.exports=t(40)},function(e){e.exports=JSON.parse('{"docusaurus-plugin-content-docs":{"default":{"path":"/netsage-pipeline/docs","versions":[{"name":"current","label":"master (unreleased)","isLast":false,"path":"/netsage-pipeline/docs/next","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/next/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/next/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/next/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/next/deploy/bare_metal_install","sidebar":"Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/next/deploy/choose_install","sidebar":"Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/next/deploy/docker_install_advanced","sidebar":"Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/next/deploy/docker_install_simple","sidebar":"Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/next/deploy/docker_troubleshoot","sidebar":"Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/next/deploy/docker_upgrade","sidebar":"Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/next/devel/dev_dataset","sidebar":"Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/next/devel/docker_dev_guide","sidebar":"Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/next/devel/docker_dev_tag","sidebar":"Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/next/devel/docusaurus","sidebar":"Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/next/pipeline/elastic","sidebar":"Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/next/pipeline/importer","sidebar":"Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/next/pipeline/intro","sidebar":"Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/next/pipeline/logstash","sidebar":"Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/next/pipeline/nfdump","sidebar":"Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/next/pipeline/tstat","sidebar":"Pipeline"}]},{"name":"2.0.0","label":"2.0.0","isLast":true,"path":"/netsage-pipeline/docs","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/deploy/bare_metal_install","sidebar":"version-2.0.0/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/deploy/choose_install","sidebar":"version-2.0.0/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/deploy/docker_install_advanced","sidebar":"version-2.0.0/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/deploy/docker_install_simple","sidebar":"version-2.0.0/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/deploy/docker_troubleshoot","sidebar":"version-2.0.0/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/deploy/docker_upgrade","sidebar":"version-2.0.0/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/devel/dev_dataset","sidebar":"version-2.0.0/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/devel/docker_dev_guide","sidebar":"version-2.0.0/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/devel/docker_dev_tag","sidebar":"version-2.0.0/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/devel/docusaurus","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/pipeline/elastic","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/pipeline/importer","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/pipeline/intro","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/pipeline/logstash","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/pipeline/nfdump","sidebar":"version-2.0.0/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/pipeline/tstat","sidebar":"version-2.0.0/Pipeline"}]},{"name":"1.2.11","label":"1.2.11","isLast":false,"path":"/netsage-pipeline/docs/1.2.11","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.11/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.11/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.11/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.11/deploy/choose_install","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot","sidebar":"version-1.2.11/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.11/devel/dev_dataset","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag","sidebar":"version-1.2.11/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/1.2.11/devel/docusaurus","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.11/pipeline/elastic","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.11/pipeline/importer","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.11/pipeline/intro","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.11/pipeline/logstash","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.11/pipeline/nfdump","sidebar":"version-1.2.11/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.11/pipeline/tstat","sidebar":"version-1.2.11/Pipeline"}]},{"name":"1.2.10","label":"1.2.10","isLast":false,"path":"/netsage-pipeline/docs/1.2.10","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.10/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.10/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.10/components/docker_pipeline"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.10/deploy/choose_install","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot","sidebar":"version-1.2.10/Pipeline"},{"id":"deploy/docker_upgrade","path":"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.10/devel/dev_dataset","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docker_dev_tag","path":"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag","sidebar":"version-1.2.10/Pipeline"},{"id":"devel/docusaurus","path":"/netsage-pipeline/docs/1.2.10/devel/docusaurus","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.10/pipeline/elastic","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.10/pipeline/importer","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.10/pipeline/intro","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.10/pipeline/logstash","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.10/pipeline/nfdump","sidebar":"version-1.2.10/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.10/pipeline/tstat","sidebar":"version-1.2.10/Pipeline"}]},{"name":"1.2.9","label":"1.2.9","isLast":false,"path":"/netsage-pipeline/docs/1.2.9","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.9/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.9/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.9/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.9/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.9/deploy/choose_install","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple","sidebar":"version-1.2.9/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot","sidebar":"version-1.2.9/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.9/devel/dev_dataset","sidebar":"version-1.2.9/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide","sidebar":"version-1.2.9/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.9/docusaurus","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.9/pipeline/elastic","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.9/pipeline/importer","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.9/pipeline/intro","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.9/pipeline/logstash","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.9/pipeline/nfdump","sidebar":"version-1.2.9/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.9/pipeline/tstat","sidebar":"version-1.2.9/Pipeline"}]},{"name":"1.2.8","label":"1.2.8","isLast":false,"path":"/netsage-pipeline/docs/1.2.8","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.8/components/docker_env"},{"id":"components/docker_first_steps","path":"/netsage-pipeline/docs/1.2.8/components/docker_first_steps"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.8/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.8/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.8/deploy/choose_install","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple","sidebar":"version-1.2.8/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot","sidebar":"version-1.2.8/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.8/devel/dev_dataset","sidebar":"version-1.2.8/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide","sidebar":"version-1.2.8/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.8/docusaurus","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.8/pipeline/elastic","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.8/pipeline/importer","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.8/pipeline/intro","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.8/pipeline/logstash","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.8/pipeline/nfdump","sidebar":"version-1.2.8/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.8/pipeline/tstat","sidebar":"version-1.2.8/Pipeline"}]},{"name":"1.2.7","label":"1.2.7","isLast":false,"path":"/netsage-pipeline/docs/1.2.7","mainDocId":"pipeline/intro","docs":[{"id":"components/docker_env","path":"/netsage-pipeline/docs/1.2.7/components/docker_env"},{"id":"components/docker_pipeline","path":"/netsage-pipeline/docs/1.2.7/components/docker_pipeline"},{"id":"components/docker_upgrade","path":"/netsage-pipeline/docs/1.2.7/components/docker_upgrade"},{"id":"deploy/bare_metal_install","path":"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.7/deploy/choose_install","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_install_advanced","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_install_simple","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple","sidebar":"version-1.2.7/Pipeline"},{"id":"deploy/docker_troubleshoot","path":"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot","sidebar":"version-1.2.7/Pipeline"},{"id":"devel/dev_dataset","path":"/netsage-pipeline/docs/1.2.7/devel/dev_dataset","sidebar":"version-1.2.7/Pipeline"},{"id":"devel/docker_dev_guide","path":"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide","sidebar":"version-1.2.7/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.7/docusaurus","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/elastic","path":"/netsage-pipeline/docs/1.2.7/pipeline/elastic","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/importer","path":"/netsage-pipeline/docs/1.2.7/pipeline/importer","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/intro","path":"/netsage-pipeline/docs/1.2.7/pipeline/intro","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/logstash","path":"/netsage-pipeline/docs/1.2.7/pipeline/logstash","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/nfdump","path":"/netsage-pipeline/docs/1.2.7/pipeline/nfdump","sidebar":"version-1.2.7/Pipeline"},{"id":"pipeline/tstat","path":"/netsage-pipeline/docs/1.2.7/pipeline/tstat","sidebar":"version-1.2.7/Pipeline"}]},{"name":"1.2.6","label":"1.2.6","isLast":false,"path":"/netsage-pipeline/docs/1.2.6","mainDocId":"pipeline","docs":[{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.6/deploy/choose_install","sidebar":"version-1.2.6/Pipeline"},{"id":"deploy/docker_install","path":"/netsage-pipeline/docs/1.2.6/deploy/docker_install","sidebar":"version-1.2.6/Pipeline"},{"id":"deploy/install","path":"/netsage-pipeline/docs/1.2.6/deploy/install","sidebar":"version-1.2.6/Pipeline"},{"id":"devel/docker","path":"/netsage-pipeline/docs/1.2.6/devel/docker","sidebar":"version-1.2.6/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.6/docusaurus","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline","path":"/netsage-pipeline/docs/1.2.6/pipeline","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline_importer","path":"/netsage-pipeline/docs/1.2.6/pipeline_importer","sidebar":"version-1.2.6/Pipeline"},{"id":"pipeline_logstash","path":"/netsage-pipeline/docs/1.2.6/pipeline_logstash","sidebar":"version-1.2.6/Pipeline"}]},{"name":"1.2.5","label":"1.2.5","isLast":false,"path":"/netsage-pipeline/docs/1.2.5","mainDocId":"pipeline","docs":[{"id":"deploy/choose_install","path":"/netsage-pipeline/docs/1.2.5/deploy/choose_install","sidebar":"version-1.2.5/Pipeline"},{"id":"deploy/docker_install","path":"/netsage-pipeline/docs/1.2.5/deploy/docker_install","sidebar":"version-1.2.5/Pipeline"},{"id":"deploy/install","path":"/netsage-pipeline/docs/1.2.5/deploy/install","sidebar":"version-1.2.5/Pipeline"},{"id":"devel/docker","path":"/netsage-pipeline/docs/1.2.5/devel/docker","sidebar":"version-1.2.5/Pipeline"},{"id":"docusaurus","path":"/netsage-pipeline/docs/1.2.5/docusaurus","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline","path":"/netsage-pipeline/docs/1.2.5/pipeline","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline_importer","path":"/netsage-pipeline/docs/1.2.5/pipeline_importer","sidebar":"version-1.2.5/Pipeline"},{"id":"pipeline_logstash","path":"/netsage-pipeline/docs/1.2.5/pipeline_logstash","sidebar":"version-1.2.5/Pipeline"}]}]}}}')},function(e){e.exports=JSON.parse('{"defaultLocale":"en","locales":["en"],"localeConfigs":{"en":{"label":"en","direction":"ltr"}},"currentLocale":"en"}')},function(e){e.exports=JSON.parse('{"docusaurusVersion":"2.0.0-alpha.72","siteVersion":"0.0.0","pluginVersions":{"docusaurus-plugin-content-docs":{"type":"package","name":"@docusaurus/plugin-content-docs","version":"2.0.0-alpha.72"},"docusaurus-plugin-content-blog":{"type":"package","name":"@docusaurus/plugin-content-blog","version":"2.0.0-alpha.72"},"docusaurus-plugin-content-pages":{"type":"package","name":"@docusaurus/plugin-content-pages","version":"2.0.0-alpha.72"},"docusaurus-plugin-sitemap":{"type":"package","name":"@docusaurus/plugin-sitemap","version":"2.0.0-alpha.72"},"docusaurus-theme-classic":{"type":"package","name":"@docusaurus/theme-classic","version":"2.0.0-alpha.72"}}}')},function(e,n,t){"use strict";var i=t(29),r={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},o={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},a={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},l={};function s(e){return i.isMemo(e)?a:l[e.$$typeof]||r}l[i.ForwardRef]={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},l[i.Memo]=a;var c=Object.defineProperty,u=Object.getOwnPropertyNames,p=Object.getOwnPropertySymbols,d=Object.getOwnPropertyDescriptor,f=Object.getPrototypeOf,m=Object.prototype;e.exports=function e(n,t,i){if("string"!=typeof t){if(m){var r=f(t);r&&r!==m&&e(n,r,i)}var a=u(t);p&&(a=a.concat(p(t)));for(var l=s(n),g=s(t),h=0;h<a.length;++h){var v=a[h];if(!(o[v]||i&&i[v]||g&&g[v]||l&&l[v])){var b=d(t,v);try{c(n,v,b)}catch(y){}}}}return n}},function(e,n,t){"use strict";(function(e){t.d(n,"a",(function(){return he}));var i,r,o,a,l=t(1),s=t.n(l),c=t(36),u=t.n(c),p=t(37),d=t.n(p),f=t(0),m=t.n(f),g=t(12),h=t.n(g),v="bodyAttributes",b="htmlAttributes",y="titleAttributes",k={BASE:"base",BODY:"body",HEAD:"head",HTML:"html",LINK:"link",META:"meta",NOSCRIPT:"noscript",SCRIPT:"script",STYLE:"style",TITLE:"title"},w=(Object.keys(k).map((function(e){return k[e]})),"charset"),_="cssText",x="href",E="http-equiv",S="innerHTML",T="itemprop",P="name",C="property",O="rel",A="src",R="target",N={accesskey:"accessKey",charset:"charSet",class:"className",contenteditable:"contentEditable",contextmenu:"contextMenu","http-equiv":"httpEquiv",itemprop:"itemProp",tabindex:"tabIndex"},I="defaultTitle",L="defer",D="encodeSpecialCharacters",j="onChangeClientState",M="titleTemplate",F=Object.keys(N).reduce((function(e,n){return e[N[n]]=n,e}),{}),B=[k.NOSCRIPT,k.SCRIPT,k.STYLE],z="data-react-helmet",U="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},$=function(e,n){if(!(e instanceof n))throw new TypeError("Cannot call a class as a function")},G=function(){function e(e,n){for(var t=0;t<n.length;t++){var i=n[t];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(e,i.key,i)}}return function(n,t,i){return t&&e(n.prototype,t),i&&e(n,i),n}}(),q=Object.assign||function(e){for(var n=1;n<arguments.length;n++){var t=arguments[n];for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&(e[i]=t[i])}return e},H=function(e,n){var t={};for(var i in e)n.indexOf(i)>=0||Object.prototype.hasOwnProperty.call(e,i)&&(t[i]=e[i]);return t},W=function(e,n){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!n||"object"!=typeof n&&"function"!=typeof n?e:n},V=function(e){var n=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];return!1===n?String(e):String(e).replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")},K=function(e){var n=Z(e,k.TITLE),t=Z(e,M);if(t&&n)return t.replace(/%s/g,(function(){return Array.isArray(n)?n.join(""):n}));var i=Z(e,I);return n||i||void 0},Y=function(e){return Z(e,j)||function(){}},Q=function(e,n){return n.filter((function(n){return void 0!==n[e]})).map((function(n){return n[e]})).reduce((function(e,n){return q({},e,n)}),{})},X=function(e,n){return n.filter((function(e){return void 0!==e[k.BASE]})).map((function(e){return e[k.BASE]})).reverse().reduce((function(n,t){if(!n.length)for(var i=Object.keys(t),r=0;r<i.length;r++){var o=i[r].toLowerCase();if(-1!==e.indexOf(o)&&t[o])return n.concat(t)}return n}),[])},J=function(e,n,t){var i={};return t.filter((function(n){return!!Array.isArray(n[e])||(void 0!==n[e]&&re("Helmet: "+e+' should be of type "Array". Instead found type "'+U(n[e])+'"'),!1)})).map((function(n){return n[e]})).reverse().reduce((function(e,t){var r={};t.filter((function(e){for(var t=void 0,o=Object.keys(e),a=0;a<o.length;a++){var l=o[a],s=l.toLowerCase();-1===n.indexOf(s)||t===O&&"canonical"===e[t].toLowerCase()||s===O&&"stylesheet"===e[s].toLowerCase()||(t=s),-1===n.indexOf(l)||l!==S&&l!==_&&l!==T||(t=l)}if(!t||!e[t])return!1;var c=e[t].toLowerCase();return i[t]||(i[t]={}),r[t]||(r[t]={}),!i[t][c]&&(r[t][c]=!0,!0)})).reverse().forEach((function(n){return e.push(n)}));for(var o=Object.keys(r),a=0;a<o.length;a++){var l=o[a],s=h()({},i[l],r[l]);i[l]=s}return e}),[]).reverse()},Z=function(e,n){for(var t=e.length-1;t>=0;t--){var i=e[t];if(i.hasOwnProperty(n))return i[n]}return null},ee=(i=Date.now(),function(e){var n=Date.now();n-i>16?(i=n,e(n)):setTimeout((function(){ee(e)}),0)}),ne=function(e){return clearTimeout(e)},te="undefined"!=typeof window?window.requestAnimationFrame&&window.requestAnimationFrame.bind(window)||window.webkitRequestAnimationFrame||window.mozRequestAnimationFrame||ee:e.requestAnimationFrame||ee,ie="undefined"!=typeof window?window.cancelAnimationFrame||window.webkitCancelAnimationFrame||window.mozCancelAnimationFrame||ne:e.cancelAnimationFrame||ne,re=function(e){return console&&"function"==typeof console.warn&&console.warn(e)},oe=null,ae=function(e,n){var t=e.baseTag,i=e.bodyAttributes,r=e.htmlAttributes,o=e.linkTags,a=e.metaTags,l=e.noscriptTags,s=e.onChangeClientState,c=e.scriptTags,u=e.styleTags,p=e.title,d=e.titleAttributes;ce(k.BODY,i),ce(k.HTML,r),se(p,d);var f={baseTag:ue(k.BASE,t),linkTags:ue(k.LINK,o),metaTags:ue(k.META,a),noscriptTags:ue(k.NOSCRIPT,l),scriptTags:ue(k.SCRIPT,c),styleTags:ue(k.STYLE,u)},m={},g={};Object.keys(f).forEach((function(e){var n=f[e],t=n.newTags,i=n.oldTags;t.length&&(m[e]=t),i.length&&(g[e]=f[e].oldTags)})),n&&n(),s(e,m,g)},le=function(e){return Array.isArray(e)?e.join(""):e},se=function(e,n){void 0!==e&&document.title!==e&&(document.title=le(e)),ce(k.TITLE,n)},ce=function(e,n){var t=document.getElementsByTagName(e)[0];if(t){for(var i=t.getAttribute(z),r=i?i.split(","):[],o=[].concat(r),a=Object.keys(n),l=0;l<a.length;l++){var s=a[l],c=n[s]||"";t.getAttribute(s)!==c&&t.setAttribute(s,c),-1===r.indexOf(s)&&r.push(s);var u=o.indexOf(s);-1!==u&&o.splice(u,1)}for(var p=o.length-1;p>=0;p--)t.removeAttribute(o[p]);r.length===o.length?t.removeAttribute(z):t.getAttribute(z)!==a.join(",")&&t.setAttribute(z,a.join(","))}},ue=function(e,n){var t=document.head||document.querySelector(k.HEAD),i=t.querySelectorAll(e+"["+"data-react-helmet]"),r=Array.prototype.slice.call(i),o=[],a=void 0;return n&&n.length&&n.forEach((function(n){var t=document.createElement(e);for(var i in n)if(n.hasOwnProperty(i))if(i===S)t.innerHTML=n.innerHTML;else if(i===_)t.styleSheet?t.styleSheet.cssText=n.cssText:t.appendChild(document.createTextNode(n.cssText));else{var l=void 0===n[i]?"":n[i];t.setAttribute(i,l)}t.setAttribute(z,"true"),r.some((function(e,n){return a=n,t.isEqualNode(e)}))?r.splice(a,1):o.push(t)})),r.forEach((function(e){return e.parentNode.removeChild(e)})),o.forEach((function(e){return t.appendChild(e)})),{oldTags:r,newTags:o}},pe=function(e){return Object.keys(e).reduce((function(n,t){var i=void 0!==e[t]?t+'="'+e[t]+'"':""+t;return n?n+" "+i:i}),"")},de=function(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return Object.keys(e).reduce((function(n,t){return n[N[t]||t]=e[t],n}),n)},fe=function(e,n,t){switch(e){case k.TITLE:return{toComponent:function(){return e=n.title,t=n.titleAttributes,(i={key:e})[z]=!0,r=de(t,i),[m.a.createElement(k.TITLE,r,e)];var e,t,i,r},toString:function(){return function(e,n,t,i){var r=pe(t),o=le(n);return r?"<"+e+' data-react-helmet="true" '+r+">"+V(o,i)+"</"+e+">":"<"+e+' data-react-helmet="true">'+V(o,i)+"</"+e+">"}(e,n.title,n.titleAttributes,t)}};case v:case b:return{toComponent:function(){return de(n)},toString:function(){return pe(n)}};default:return{toComponent:function(){return function(e,n){return n.map((function(n,t){var i,r=((i={key:t})[z]=!0,i);return Object.keys(n).forEach((function(e){var t=N[e]||e;if(t===S||t===_){var i=n.innerHTML||n.cssText;r.dangerouslySetInnerHTML={__html:i}}else r[t]=n[e]})),m.a.createElement(e,r)}))}(e,n)},toString:function(){return function(e,n,t){return n.reduce((function(n,i){var r=Object.keys(i).filter((function(e){return!(e===S||e===_)})).reduce((function(e,n){var r=void 0===i[n]?n:n+'="'+V(i[n],t)+'"';return e?e+" "+r:r}),""),o=i.innerHTML||i.cssText||"",a=-1===B.indexOf(e);return n+"<"+e+' data-react-helmet="true" '+r+(a?"/>":">"+o+"</"+e+">")}),"")}(e,n,t)}}}},me=function(e){var n=e.baseTag,t=e.bodyAttributes,i=e.encode,r=e.htmlAttributes,o=e.linkTags,a=e.metaTags,l=e.noscriptTags,s=e.scriptTags,c=e.styleTags,u=e.title,p=void 0===u?"":u,d=e.titleAttributes;return{base:fe(k.BASE,n,i),bodyAttributes:fe(v,t,i),htmlAttributes:fe(b,r,i),link:fe(k.LINK,o,i),meta:fe(k.META,a,i),noscript:fe(k.NOSCRIPT,l,i),script:fe(k.SCRIPT,s,i),style:fe(k.STYLE,c,i),title:fe(k.TITLE,{title:p,titleAttributes:d},i)}},ge=u()((function(e){return{baseTag:X([x,R],e),bodyAttributes:Q(v,e),defer:Z(e,L),encode:Z(e,D),htmlAttributes:Q(b,e),linkTags:J(k.LINK,[O,x],e),metaTags:J(k.META,[P,w,E,C,T],e),noscriptTags:J(k.NOSCRIPT,[S],e),onChangeClientState:Y(e),scriptTags:J(k.SCRIPT,[A,S],e),styleTags:J(k.STYLE,[_],e),title:K(e),titleAttributes:Q(y,e)}}),(function(e){oe&&ie(oe),e.defer?oe=te((function(){ae(e,(function(){oe=null}))})):(ae(e),oe=null)}),me)((function(){return null})),he=(r=ge,a=o=function(e){function n(){return $(this,n),W(this,e.apply(this,arguments))}return function(e,n){if("function"!=typeof n&&null!==n)throw new TypeError("Super expression must either be null or a function, not "+typeof n);e.prototype=Object.create(n&&n.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),n&&(Object.setPrototypeOf?Object.setPrototypeOf(e,n):e.__proto__=n)}(n,e),n.prototype.shouldComponentUpdate=function(e){return!d()(this.props,e)},n.prototype.mapNestedChildrenToProps=function(e,n){if(!n)return null;switch(e.type){case k.SCRIPT:case k.NOSCRIPT:return{innerHTML:n};case k.STYLE:return{cssText:n}}throw new Error("<"+e.type+" /> elements are self-closing and can not contain children. Refer to our API for more information.")},n.prototype.flattenArrayTypeChildren=function(e){var n,t=e.child,i=e.arrayTypeChildren,r=e.newChildProps,o=e.nestedChildren;return q({},i,((n={})[t.type]=[].concat(i[t.type]||[],[q({},r,this.mapNestedChildrenToProps(t,o))]),n))},n.prototype.mapObjectTypeChildren=function(e){var n,t,i=e.child,r=e.newProps,o=e.newChildProps,a=e.nestedChildren;switch(i.type){case k.TITLE:return q({},r,((n={})[i.type]=a,n.titleAttributes=q({},o),n));case k.BODY:return q({},r,{bodyAttributes:q({},o)});case k.HTML:return q({},r,{htmlAttributes:q({},o)})}return q({},r,((t={})[i.type]=q({},o),t))},n.prototype.mapArrayTypeChildrenToProps=function(e,n){var t=q({},n);return Object.keys(e).forEach((function(n){var i;t=q({},t,((i={})[n]=e[n],i))})),t},n.prototype.warnOnInvalidChildren=function(e,n){return!0},n.prototype.mapChildrenToProps=function(e,n){var t=this,i={};return m.a.Children.forEach(e,(function(e){if(e&&e.props){var r=e.props,o=r.children,a=function(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return Object.keys(e).reduce((function(n,t){return n[F[t]||t]=e[t],n}),n)}(H(r,["children"]));switch(t.warnOnInvalidChildren(e,o),e.type){case k.LINK:case k.META:case k.NOSCRIPT:case k.SCRIPT:case k.STYLE:i=t.flattenArrayTypeChildren({child:e,arrayTypeChildren:i,newChildProps:a,nestedChildren:o});break;default:n=t.mapObjectTypeChildren({child:e,newProps:n,newChildProps:a,nestedChildren:o})}}})),n=this.mapArrayTypeChildrenToProps(i,n)},n.prototype.render=function(){var e=this.props,n=e.children,t=H(e,["children"]),i=q({},t);return n&&(i=this.mapChildrenToProps(n,i)),m.a.createElement(r,i)},G(n,null,[{key:"canUseDOM",set:function(e){r.canUseDOM=e}}]),n}(m.a.Component),o.propTypes={base:s.a.object,bodyAttributes:s.a.object,children:s.a.oneOfType([s.a.arrayOf(s.a.node),s.a.node]),defaultTitle:s.a.string,defer:s.a.bool,encodeSpecialCharacters:s.a.bool,htmlAttributes:s.a.object,link:s.a.arrayOf(s.a.object),meta:s.a.arrayOf(s.a.object),noscript:s.a.arrayOf(s.a.object),onChangeClientState:s.a.func,script:s.a.arrayOf(s.a.object),style:s.a.arrayOf(s.a.object),title:s.a.string,titleAttributes:s.a.object,titleTemplate:s.a.string},o.defaultProps={defer:!0,encodeSpecialCharacters:!0},o.peek=r.peek,o.rewind=function(){var e=r.rewind();return e||(e=me({baseTag:[],bodyAttributes:{},encodeSpecialCharacters:!0,htmlAttributes:{},linkTags:[],metaTags:[],noscriptTags:[],scriptTags:[],styleTags:[],title:"",titleAttributes:{}})),e},a);he.renderStatic=he.rewind}).call(this,t(28))},function(e,n,t){"use strict";var i,r=t(0),o=(i=r)&&"object"==typeof i&&"default"in i?i.default:i;function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}var l=!("undefined"==typeof window||!window.document||!window.document.createElement);e.exports=function(e,n,t){if("function"!=typeof e)throw new Error("Expected reducePropsToState to be a function.");if("function"!=typeof n)throw new Error("Expected handleStateChangeOnClient to be a function.");if(void 0!==t&&"function"!=typeof t)throw new Error("Expected mapStateOnServer to either be undefined or a function.");return function(i){if("function"!=typeof i)throw new Error("Expected WrappedComponent to be a React component.");var s,c=[];function u(){s=e(c.map((function(e){return e.props}))),p.canUseDOM?n(s):t&&(s=t(s))}var p=function(e){var n,t;function r(){return e.apply(this,arguments)||this}t=e,(n=r).prototype=Object.create(t.prototype),n.prototype.constructor=n,n.__proto__=t,r.peek=function(){return s},r.rewind=function(){if(r.canUseDOM)throw new Error("You may only call rewind() on the server. Call peek() to read the current state.");var e=s;return s=void 0,c=[],e};var a=r.prototype;return a.UNSAFE_componentWillMount=function(){c.push(this),u()},a.componentDidUpdate=function(){u()},a.componentWillUnmount=function(){var e=c.indexOf(this);c.splice(e,1),u()},a.render=function(){return o.createElement(i,this.props)},r}(r.PureComponent);return a(p,"displayName","SideEffect("+function(e){return e.displayName||e.name||"Component"}(i)+")"),a(p,"canUseDOM",l),p}}},function(e,n){var t="undefined"!=typeof Element,i="function"==typeof Map,r="function"==typeof Set,o="function"==typeof ArrayBuffer&&!!ArrayBuffer.isView;function a(e,n){if(e===n)return!0;if(e&&n&&"object"==typeof e&&"object"==typeof n){if(e.constructor!==n.constructor)return!1;var l,s,c,u;if(Array.isArray(e)){if((l=e.length)!=n.length)return!1;for(s=l;0!=s--;)if(!a(e[s],n[s]))return!1;return!0}if(i&&e instanceof Map&&n instanceof Map){if(e.size!==n.size)return!1;for(u=e.entries();!(s=u.next()).done;)if(!n.has(s.value[0]))return!1;for(u=e.entries();!(s=u.next()).done;)if(!a(s.value[1],n.get(s.value[0])))return!1;return!0}if(r&&e instanceof Set&&n instanceof Set){if(e.size!==n.size)return!1;for(u=e.entries();!(s=u.next()).done;)if(!n.has(s.value[0]))return!1;return!0}if(o&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(n)){if((l=e.length)!=n.length)return!1;for(s=l;0!=s--;)if(e[s]!==n[s])return!1;return!0}if(e.constructor===RegExp)return e.source===n.source&&e.flags===n.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===n.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===n.toString();if((l=(c=Object.keys(e)).length)!==Object.keys(n).length)return!1;for(s=l;0!=s--;)if(!Object.prototype.hasOwnProperty.call(n,c[s]))return!1;if(t&&e instanceof Element)return!1;for(s=l;0!=s--;)if(("_owner"!==c[s]&&"__v"!==c[s]&&"__o"!==c[s]||!e.$$typeof)&&!a(e[c[s]],n[c[s]]))return!1;return!0}return e!=e&&n!=n}e.exports=function(e,n){try{return a(e,n)}catch(t){if((t.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw t}}},function(e,n,t){e.exports=t(54)},function(e,n,t){"use strict";var i=t(12),r="function"==typeof Symbol&&Symbol.for,o=r?Symbol.for("react.element"):60103,a=r?Symbol.for("react.portal"):60106,l=r?Symbol.for("react.fragment"):60107,s=r?Symbol.for("react.strict_mode"):60108,c=r?Symbol.for("react.profiler"):60114,u=r?Symbol.for("react.provider"):60109,p=r?Symbol.for("react.context"):60110,d=r?Symbol.for("react.forward_ref"):60112,f=r?Symbol.for("react.suspense"):60113,m=r?Symbol.for("react.memo"):60115,g=r?Symbol.for("react.lazy"):60116,h="function"==typeof Symbol&&Symbol.iterator;function v(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t<arguments.length;t++)n+="&args[]="+encodeURIComponent(arguments[t]);return"Minified React error #"+e+"; visit "+n+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}var b={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},y={};function k(e,n,t){this.props=e,this.context=n,this.refs=y,this.updater=t||b}function w(){}function _(e,n,t){this.props=e,this.context=n,this.refs=y,this.updater=t||b}k.prototype.isReactComponent={},k.prototype.setState=function(e,n){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error(v(85));this.updater.enqueueSetState(this,e,n,"setState")},k.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},w.prototype=k.prototype;var x=_.prototype=new w;x.constructor=_,i(x,k.prototype),x.isPureReactComponent=!0;var E={current:null},S=Object.prototype.hasOwnProperty,T={key:!0,ref:!0,__self:!0,__source:!0};function P(e,n,t){var i,r={},a=null,l=null;if(null!=n)for(i in void 0!==n.ref&&(l=n.ref),void 0!==n.key&&(a=""+n.key),n)S.call(n,i)&&!T.hasOwnProperty(i)&&(r[i]=n[i]);var s=arguments.length-2;if(1===s)r.children=t;else if(1<s){for(var c=Array(s),u=0;u<s;u++)c[u]=arguments[u+2];r.children=c}if(e&&e.defaultProps)for(i in s=e.defaultProps)void 0===r[i]&&(r[i]=s[i]);return{$$typeof:o,type:e,key:a,ref:l,props:r,_owner:E.current}}function C(e){return"object"==typeof e&&null!==e&&e.$$typeof===o}var O=/\/+/g,A=[];function R(e,n,t,i){if(A.length){var r=A.pop();return r.result=e,r.keyPrefix=n,r.func=t,r.context=i,r.count=0,r}return{result:e,keyPrefix:n,func:t,context:i,count:0}}function N(e){e.result=null,e.keyPrefix=null,e.func=null,e.context=null,e.count=0,10>A.length&&A.push(e)}function I(e,n,t,i){var r=typeof e;"undefined"!==r&&"boolean"!==r||(e=null);var l=!1;if(null===e)l=!0;else switch(r){case"string":case"number":l=!0;break;case"object":switch(e.$$typeof){case o:case a:l=!0}}if(l)return t(i,e,""===n?"."+D(e,0):n),1;if(l=0,n=""===n?".":n+":",Array.isArray(e))for(var s=0;s<e.length;s++){var c=n+D(r=e[s],s);l+=I(r,c,t,i)}else if(null===e||"object"!=typeof e?c=null:c="function"==typeof(c=h&&e[h]||e["@@iterator"])?c:null,"function"==typeof c)for(e=c.call(e),s=0;!(r=e.next()).done;)l+=I(r=r.value,c=n+D(r,s++),t,i);else if("object"===r)throw t=""+e,Error(v(31,"[object Object]"===t?"object with keys {"+Object.keys(e).join(", ")+"}":t,""));return l}function L(e,n,t){return null==e?0:I(e,"",n,t)}function D(e,n){return"object"==typeof e&&null!==e&&null!=e.key?function(e){var n={"=":"=0",":":"=2"};return"$"+(""+e).replace(/[=:]/g,(function(e){return n[e]}))}(e.key):n.toString(36)}function j(e,n){e.func.call(e.context,n,e.count++)}function M(e,n,t){var i=e.result,r=e.keyPrefix;e=e.func.call(e.context,n,e.count++),Array.isArray(e)?F(e,i,t,(function(e){return e})):null!=e&&(C(e)&&(e=function(e,n){return{$$typeof:o,type:e.type,key:n,ref:e.ref,props:e.props,_owner:e._owner}}(e,r+(!e.key||n&&n.key===e.key?"":(""+e.key).replace(O,"$&/")+"/")+t)),i.push(e))}function F(e,n,t,i,r){var o="";null!=t&&(o=(""+t).replace(O,"$&/")+"/"),L(e,M,n=R(n,o,i,r)),N(n)}var B={current:null};function z(){var e=B.current;if(null===e)throw Error(v(321));return e}var U={ReactCurrentDispatcher:B,ReactCurrentBatchConfig:{suspense:null},ReactCurrentOwner:E,IsSomeRendererActing:{current:!1},assign:i};n.Children={map:function(e,n,t){if(null==e)return e;var i=[];return F(e,i,null,n,t),i},forEach:function(e,n,t){if(null==e)return e;L(e,j,n=R(null,null,n,t)),N(n)},count:function(e){return L(e,(function(){return null}),null)},toArray:function(e){var n=[];return F(e,n,null,(function(e){return e})),n},only:function(e){if(!C(e))throw Error(v(143));return e}},n.Component=k,n.Fragment=l,n.Profiler=c,n.PureComponent=_,n.StrictMode=s,n.Suspense=f,n.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=U,n.cloneElement=function(e,n,t){if(null==e)throw Error(v(267,e));var r=i({},e.props),a=e.key,l=e.ref,s=e._owner;if(null!=n){if(void 0!==n.ref&&(l=n.ref,s=E.current),void 0!==n.key&&(a=""+n.key),e.type&&e.type.defaultProps)var c=e.type.defaultProps;for(u in n)S.call(n,u)&&!T.hasOwnProperty(u)&&(r[u]=void 0===n[u]&&void 0!==c?c[u]:n[u])}var u=arguments.length-2;if(1===u)r.children=t;else if(1<u){c=Array(u);for(var p=0;p<u;p++)c[p]=arguments[p+2];r.children=c}return{$$typeof:o,type:e.type,key:a,ref:l,props:r,_owner:s}},n.createContext=function(e,n){return void 0===n&&(n=null),(e={$$typeof:p,_calculateChangedBits:n,_currentValue:e,_currentValue2:e,_threadCount:0,Provider:null,Consumer:null}).Provider={$$typeof:u,_context:e},e.Consumer=e},n.createElement=P,n.createFactory=function(e){var n=P.bind(null,e);return n.type=e,n},n.createRef=function(){return{current:null}},n.forwardRef=function(e){return{$$typeof:d,render:e}},n.isValidElement=C,n.lazy=function(e){return{$$typeof:g,_ctor:e,_status:-1,_result:null}},n.memo=function(e,n){return{$$typeof:m,type:e,compare:void 0===n?null:n}},n.useCallback=function(e,n){return z().useCallback(e,n)},n.useContext=function(e,n){return z().useContext(e,n)},n.useDebugValue=function(){},n.useEffect=function(e,n){return z().useEffect(e,n)},n.useImperativeHandle=function(e,n,t){return z().useImperativeHandle(e,n,t)},n.useLayoutEffect=function(e,n){return z().useLayoutEffect(e,n)},n.useMemo=function(e,n){return z().useMemo(e,n)},n.useReducer=function(e,n,t){return z().useReducer(e,n,t)},n.useRef=function(e){return z().useRef(e)},n.useState=function(e){return z().useState(e)},n.version="16.14.0"},function(e,n,t){"use strict";var i=t(0),r=t(12),o=t(41);function a(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t<arguments.length;t++)n+="&args[]="+encodeURIComponent(arguments[t]);return"Minified React error #"+e+"; visit "+n+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}if(!i)throw Error(a(227));function l(e,n,t,i,r,o,a,l,s){var c=Array.prototype.slice.call(arguments,3);try{n.apply(t,c)}catch(u){this.onError(u)}}var s=!1,c=null,u=!1,p=null,d={onError:function(e){s=!0,c=e}};function f(e,n,t,i,r,o,a,u,p){s=!1,c=null,l.apply(d,arguments)}var m=null,g=null,h=null;function v(e,n,t){var i=e.type||"unknown-event";e.currentTarget=h(t),function(e,n,t,i,r,o,l,d,m){if(f.apply(this,arguments),s){if(!s)throw Error(a(198));var g=c;s=!1,c=null,u||(u=!0,p=g)}}(i,n,void 0,e),e.currentTarget=null}var b=null,y={};function k(){if(b)for(var e in y){var n=y[e],t=b.indexOf(e);if(!(-1<t))throw Error(a(96,e));if(!_[t]){if(!n.extractEvents)throw Error(a(97,e));for(var i in _[t]=n,t=n.eventTypes){var r=void 0,o=t[i],l=n,s=i;if(x.hasOwnProperty(s))throw Error(a(99,s));x[s]=o;var c=o.phasedRegistrationNames;if(c){for(r in c)c.hasOwnProperty(r)&&w(c[r],l,s);r=!0}else o.registrationName?(w(o.registrationName,l,s),r=!0):r=!1;if(!r)throw Error(a(98,i,e))}}}}function w(e,n,t){if(E[e])throw Error(a(100,e));E[e]=n,S[e]=n.eventTypes[t].dependencies}var _=[],x={},E={},S={};function T(e){var n,t=!1;for(n in e)if(e.hasOwnProperty(n)){var i=e[n];if(!y.hasOwnProperty(n)||y[n]!==i){if(y[n])throw Error(a(102,n));y[n]=i,t=!0}}t&&k()}var P=!("undefined"==typeof window||void 0===window.document||void 0===window.document.createElement),C=null,O=null,A=null;function R(e){if(e=g(e)){if("function"!=typeof C)throw Error(a(280));var n=e.stateNode;n&&(n=m(n),C(e.stateNode,e.type,n))}}function N(e){O?A?A.push(e):A=[e]:O=e}function I(){if(O){var e=O,n=A;if(A=O=null,R(e),n)for(e=0;e<n.length;e++)R(n[e])}}function L(e,n){return e(n)}function D(e,n,t,i,r){return e(n,t,i,r)}function j(){}var M=L,F=!1,B=!1;function z(){null===O&&null===A||(j(),I())}function U(e,n,t){if(B)return e(n,t);B=!0;try{return M(e,n,t)}finally{B=!1,z()}}var $=/^[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD][:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-.0-9\u00B7\u0300-\u036F\u203F-\u2040]*$/,G=Object.prototype.hasOwnProperty,q={},H={};function W(e,n,t,i,r,o){this.acceptsBooleans=2===n||3===n||4===n,this.attributeName=i,this.attributeNamespace=r,this.mustUseProperty=t,this.propertyName=e,this.type=n,this.sanitizeURL=o}var V={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach((function(e){V[e]=new W(e,0,!1,e,null,!1)})),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach((function(e){var n=e[0];V[n]=new W(n,1,!1,e[1],null,!1)})),["contentEditable","draggable","spellCheck","value"].forEach((function(e){V[e]=new W(e,2,!1,e.toLowerCase(),null,!1)})),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach((function(e){V[e]=new W(e,2,!1,e,null,!1)})),"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach((function(e){V[e]=new W(e,3,!1,e.toLowerCase(),null,!1)})),["checked","multiple","muted","selected"].forEach((function(e){V[e]=new W(e,3,!0,e,null,!1)})),["capture","download"].forEach((function(e){V[e]=new W(e,4,!1,e,null,!1)})),["cols","rows","size","span"].forEach((function(e){V[e]=new W(e,6,!1,e,null,!1)})),["rowSpan","start"].forEach((function(e){V[e]=new W(e,5,!1,e.toLowerCase(),null,!1)}));var K=/[\-:]([a-z])/g;function Y(e){return e[1].toUpperCase()}"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,null,!1)})),"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,"http://www.w3.org/1999/xlink",!1)})),["xml:base","xml:lang","xml:space"].forEach((function(e){var n=e.replace(K,Y);V[n]=new W(n,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1)})),["tabIndex","crossOrigin"].forEach((function(e){V[e]=new W(e,1,!1,e.toLowerCase(),null,!1)})),V.xlinkHref=new W("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0),["src","href","action","formAction"].forEach((function(e){V[e]=new W(e,1,!1,e.toLowerCase(),null,!0)}));var Q=i.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED;function X(e,n,t,i){var r=V.hasOwnProperty(n)?V[n]:null;(null!==r?0===r.type:!i&&(2<n.length&&("o"===n[0]||"O"===n[0])&&("n"===n[1]||"N"===n[1])))||(function(e,n,t,i){if(null==n||function(e,n,t,i){if(null!==t&&0===t.type)return!1;switch(typeof n){case"function":case"symbol":return!0;case"boolean":return!i&&(null!==t?!t.acceptsBooleans:"data-"!==(e=e.toLowerCase().slice(0,5))&&"aria-"!==e);default:return!1}}(e,n,t,i))return!0;if(i)return!1;if(null!==t)switch(t.type){case 3:return!n;case 4:return!1===n;case 5:return isNaN(n);case 6:return isNaN(n)||1>n}return!1}(n,t,r,i)&&(t=null),i||null===r?function(e){return!!G.call(H,e)||!G.call(q,e)&&($.test(e)?H[e]=!0:(q[e]=!0,!1))}(n)&&(null===t?e.removeAttribute(n):e.setAttribute(n,""+t)):r.mustUseProperty?e[r.propertyName]=null===t?3!==r.type&&"":t:(n=r.attributeName,i=r.attributeNamespace,null===t?e.removeAttribute(n):(t=3===(r=r.type)||4===r&&!0===t?"":""+t,i?e.setAttributeNS(i,n,t):e.setAttribute(n,t))))}Q.hasOwnProperty("ReactCurrentDispatcher")||(Q.ReactCurrentDispatcher={current:null}),Q.hasOwnProperty("ReactCurrentBatchConfig")||(Q.ReactCurrentBatchConfig={suspense:null});var J=/^(.*)[\\\/]/,Z="function"==typeof Symbol&&Symbol.for,ee=Z?Symbol.for("react.element"):60103,ne=Z?Symbol.for("react.portal"):60106,te=Z?Symbol.for("react.fragment"):60107,ie=Z?Symbol.for("react.strict_mode"):60108,re=Z?Symbol.for("react.profiler"):60114,oe=Z?Symbol.for("react.provider"):60109,ae=Z?Symbol.for("react.context"):60110,le=Z?Symbol.for("react.concurrent_mode"):60111,se=Z?Symbol.for("react.forward_ref"):60112,ce=Z?Symbol.for("react.suspense"):60113,ue=Z?Symbol.for("react.suspense_list"):60120,pe=Z?Symbol.for("react.memo"):60115,de=Z?Symbol.for("react.lazy"):60116,fe=Z?Symbol.for("react.block"):60121,me="function"==typeof Symbol&&Symbol.iterator;function ge(e){return null===e||"object"!=typeof e?null:"function"==typeof(e=me&&e[me]||e["@@iterator"])?e:null}function he(e){if(null==e)return null;if("function"==typeof e)return e.displayName||e.name||null;if("string"==typeof e)return e;switch(e){case te:return"Fragment";case ne:return"Portal";case re:return"Profiler";case ie:return"StrictMode";case ce:return"Suspense";case ue:return"SuspenseList"}if("object"==typeof e)switch(e.$$typeof){case ae:return"Context.Consumer";case oe:return"Context.Provider";case se:var n=e.render;return n=n.displayName||n.name||"",e.displayName||(""!==n?"ForwardRef("+n+")":"ForwardRef");case pe:return he(e.type);case fe:return he(e.render);case de:if(e=1===e._status?e._result:null)return he(e)}return null}function ve(e){var n="";do{e:switch(e.tag){case 3:case 4:case 6:case 7:case 10:case 9:var t="";break e;default:var i=e._debugOwner,r=e._debugSource,o=he(e.type);t=null,i&&(t=he(i.type)),i=o,o="",r?o=" (at "+r.fileName.replace(J,"")+":"+r.lineNumber+")":t&&(o=" (created by "+t+")"),t="\n    in "+(i||"Unknown")+o}n+=t,e=e.return}while(e);return n}function be(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}function ye(e){var n=e.type;return(e=e.nodeName)&&"input"===e.toLowerCase()&&("checkbox"===n||"radio"===n)}function ke(e){e._valueTracker||(e._valueTracker=function(e){var n=ye(e)?"checked":"value",t=Object.getOwnPropertyDescriptor(e.constructor.prototype,n),i=""+e[n];if(!e.hasOwnProperty(n)&&void 0!==t&&"function"==typeof t.get&&"function"==typeof t.set){var r=t.get,o=t.set;return Object.defineProperty(e,n,{configurable:!0,get:function(){return r.call(this)},set:function(e){i=""+e,o.call(this,e)}}),Object.defineProperty(e,n,{enumerable:t.enumerable}),{getValue:function(){return i},setValue:function(e){i=""+e},stopTracking:function(){e._valueTracker=null,delete e[n]}}}}(e))}function we(e){if(!e)return!1;var n=e._valueTracker;if(!n)return!0;var t=n.getValue(),i="";return e&&(i=ye(e)?e.checked?"true":"false":e.value),(e=i)!==t&&(n.setValue(e),!0)}function _e(e,n){var t=n.checked;return r({},n,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:null!=t?t:e._wrapperState.initialChecked})}function xe(e,n){var t=null==n.defaultValue?"":n.defaultValue,i=null!=n.checked?n.checked:n.defaultChecked;t=be(null!=n.value?n.value:t),e._wrapperState={initialChecked:i,initialValue:t,controlled:"checkbox"===n.type||"radio"===n.type?null!=n.checked:null!=n.value}}function Ee(e,n){null!=(n=n.checked)&&X(e,"checked",n,!1)}function Se(e,n){Ee(e,n);var t=be(n.value),i=n.type;if(null!=t)"number"===i?(0===t&&""===e.value||e.value!=t)&&(e.value=""+t):e.value!==""+t&&(e.value=""+t);else if("submit"===i||"reset"===i)return void e.removeAttribute("value");n.hasOwnProperty("value")?Pe(e,n.type,t):n.hasOwnProperty("defaultValue")&&Pe(e,n.type,be(n.defaultValue)),null==n.checked&&null!=n.defaultChecked&&(e.defaultChecked=!!n.defaultChecked)}function Te(e,n,t){if(n.hasOwnProperty("value")||n.hasOwnProperty("defaultValue")){var i=n.type;if(!("submit"!==i&&"reset"!==i||void 0!==n.value&&null!==n.value))return;n=""+e._wrapperState.initialValue,t||n===e.value||(e.value=n),e.defaultValue=n}""!==(t=e.name)&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,""!==t&&(e.name=t)}function Pe(e,n,t){"number"===n&&e.ownerDocument.activeElement===e||(null==t?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+t&&(e.defaultValue=""+t))}function Ce(e,n){return e=r({children:void 0},n),(n=function(e){var n="";return i.Children.forEach(e,(function(e){null!=e&&(n+=e)})),n}(n.children))&&(e.children=n),e}function Oe(e,n,t,i){if(e=e.options,n){n={};for(var r=0;r<t.length;r++)n["$"+t[r]]=!0;for(t=0;t<e.length;t++)r=n.hasOwnProperty("$"+e[t].value),e[t].selected!==r&&(e[t].selected=r),r&&i&&(e[t].defaultSelected=!0)}else{for(t=""+be(t),n=null,r=0;r<e.length;r++){if(e[r].value===t)return e[r].selected=!0,void(i&&(e[r].defaultSelected=!0));null!==n||e[r].disabled||(n=e[r])}null!==n&&(n.selected=!0)}}function Ae(e,n){if(null!=n.dangerouslySetInnerHTML)throw Error(a(91));return r({},n,{value:void 0,defaultValue:void 0,children:""+e._wrapperState.initialValue})}function Re(e,n){var t=n.value;if(null==t){if(t=n.children,n=n.defaultValue,null!=t){if(null!=n)throw Error(a(92));if(Array.isArray(t)){if(!(1>=t.length))throw Error(a(93));t=t[0]}n=t}null==n&&(n=""),t=n}e._wrapperState={initialValue:be(t)}}function Ne(e,n){var t=be(n.value),i=be(n.defaultValue);null!=t&&((t=""+t)!==e.value&&(e.value=t),null==n.defaultValue&&e.defaultValue!==t&&(e.defaultValue=t)),null!=i&&(e.defaultValue=""+i)}function Ie(e){var n=e.textContent;n===e._wrapperState.initialValue&&""!==n&&null!==n&&(e.value=n)}var Le="http://www.w3.org/1999/xhtml",De="http://www.w3.org/2000/svg";function je(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}function Me(e,n){return null==e||"http://www.w3.org/1999/xhtml"===e?je(n):"http://www.w3.org/2000/svg"===e&&"foreignObject"===n?"http://www.w3.org/1999/xhtml":e}var Fe,Be,ze=(Be=function(e,n){if(e.namespaceURI!==De||"innerHTML"in e)e.innerHTML=n;else{for((Fe=Fe||document.createElement("div")).innerHTML="<svg>"+n.valueOf().toString()+"</svg>",n=Fe.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;n.firstChild;)e.appendChild(n.firstChild)}},"undefined"!=typeof MSApp&&MSApp.execUnsafeLocalFunction?function(e,n,t,i){MSApp.execUnsafeLocalFunction((function(){return Be(e,n)}))}:Be);function Ue(e,n){if(n){var t=e.firstChild;if(t&&t===e.lastChild&&3===t.nodeType)return void(t.nodeValue=n)}e.textContent=n}function $e(e,n){var t={};return t[e.toLowerCase()]=n.toLowerCase(),t["Webkit"+e]="webkit"+n,t["Moz"+e]="moz"+n,t}var Ge={animationend:$e("Animation","AnimationEnd"),animationiteration:$e("Animation","AnimationIteration"),animationstart:$e("Animation","AnimationStart"),transitionend:$e("Transition","TransitionEnd")},qe={},He={};function We(e){if(qe[e])return qe[e];if(!Ge[e])return e;var n,t=Ge[e];for(n in t)if(t.hasOwnProperty(n)&&n in He)return qe[e]=t[n];return e}P&&(He=document.createElement("div").style,"AnimationEvent"in window||(delete Ge.animationend.animation,delete Ge.animationiteration.animation,delete Ge.animationstart.animation),"TransitionEvent"in window||delete Ge.transitionend.transition);var Ve=We("animationend"),Ke=We("animationiteration"),Ye=We("animationstart"),Qe=We("transitionend"),Xe="abort canplay canplaythrough durationchange emptied encrypted ended error loadeddata loadedmetadata loadstart pause play playing progress ratechange seeked seeking stalled suspend timeupdate volumechange waiting".split(" "),Je=new("function"==typeof WeakMap?WeakMap:Map);function Ze(e){var n=Je.get(e);return void 0===n&&(n=new Map,Je.set(e,n)),n}function en(e){var n=e,t=e;if(e.alternate)for(;n.return;)n=n.return;else{e=n;do{0!=(1026&(n=e).effectTag)&&(t=n.return),e=n.return}while(e)}return 3===n.tag?t:null}function nn(e){if(13===e.tag){var n=e.memoizedState;if(null===n&&(null!==(e=e.alternate)&&(n=e.memoizedState)),null!==n)return n.dehydrated}return null}function tn(e){if(en(e)!==e)throw Error(a(188))}function rn(e){if(!(e=function(e){var n=e.alternate;if(!n){if(null===(n=en(e)))throw Error(a(188));return n!==e?null:e}for(var t=e,i=n;;){var r=t.return;if(null===r)break;var o=r.alternate;if(null===o){if(null!==(i=r.return)){t=i;continue}break}if(r.child===o.child){for(o=r.child;o;){if(o===t)return tn(r),e;if(o===i)return tn(r),n;o=o.sibling}throw Error(a(188))}if(t.return!==i.return)t=r,i=o;else{for(var l=!1,s=r.child;s;){if(s===t){l=!0,t=r,i=o;break}if(s===i){l=!0,i=r,t=o;break}s=s.sibling}if(!l){for(s=o.child;s;){if(s===t){l=!0,t=o,i=r;break}if(s===i){l=!0,i=o,t=r;break}s=s.sibling}if(!l)throw Error(a(189))}}if(t.alternate!==i)throw Error(a(190))}if(3!==t.tag)throw Error(a(188));return t.stateNode.current===t?e:n}(e)))return null;for(var n=e;;){if(5===n.tag||6===n.tag)return n;if(n.child)n.child.return=n,n=n.child;else{if(n===e)break;for(;!n.sibling;){if(!n.return||n.return===e)return null;n=n.return}n.sibling.return=n.return,n=n.sibling}}return null}function on(e,n){if(null==n)throw Error(a(30));return null==e?n:Array.isArray(e)?Array.isArray(n)?(e.push.apply(e,n),e):(e.push(n),e):Array.isArray(n)?[e].concat(n):[e,n]}function an(e,n,t){Array.isArray(e)?e.forEach(n,t):e&&n.call(t,e)}var ln=null;function sn(e){if(e){var n=e._dispatchListeners,t=e._dispatchInstances;if(Array.isArray(n))for(var i=0;i<n.length&&!e.isPropagationStopped();i++)v(e,n[i],t[i]);else n&&v(e,n,t);e._dispatchListeners=null,e._dispatchInstances=null,e.isPersistent()||e.constructor.release(e)}}function cn(e){if(null!==e&&(ln=on(ln,e)),e=ln,ln=null,e){if(an(e,sn),ln)throw Error(a(95));if(u)throw e=p,u=!1,p=null,e}}function un(e){return(e=e.target||e.srcElement||window).correspondingUseElement&&(e=e.correspondingUseElement),3===e.nodeType?e.parentNode:e}function pn(e){if(!P)return!1;var n=(e="on"+e)in document;return n||((n=document.createElement("div")).setAttribute(e,"return;"),n="function"==typeof n[e]),n}var dn=[];function fn(e){e.topLevelType=null,e.nativeEvent=null,e.targetInst=null,e.ancestors.length=0,10>dn.length&&dn.push(e)}function mn(e,n,t,i){if(dn.length){var r=dn.pop();return r.topLevelType=e,r.eventSystemFlags=i,r.nativeEvent=n,r.targetInst=t,r}return{topLevelType:e,eventSystemFlags:i,nativeEvent:n,targetInst:t,ancestors:[]}}function gn(e){var n=e.targetInst,t=n;do{if(!t){e.ancestors.push(t);break}var i=t;if(3===i.tag)i=i.stateNode.containerInfo;else{for(;i.return;)i=i.return;i=3!==i.tag?null:i.stateNode.containerInfo}if(!i)break;5!==(n=t.tag)&&6!==n||e.ancestors.push(t),t=Rt(i)}while(t);for(t=0;t<e.ancestors.length;t++){n=e.ancestors[t];var r=un(e.nativeEvent);i=e.topLevelType;var o=e.nativeEvent,a=e.eventSystemFlags;0===t&&(a|=64);for(var l=null,s=0;s<_.length;s++){var c=_[s];c&&(c=c.extractEvents(i,n,o,r,a))&&(l=on(l,c))}cn(l)}}function hn(e,n,t){if(!t.has(e)){switch(e){case"scroll":Qn(n,"scroll",!0);break;case"focus":case"blur":Qn(n,"focus",!0),Qn(n,"blur",!0),t.set("blur",null),t.set("focus",null);break;case"cancel":case"close":pn(e)&&Qn(n,e,!0);break;case"invalid":case"submit":case"reset":break;default:-1===Xe.indexOf(e)&&Yn(e,n)}t.set(e,null)}}var vn,bn,yn,kn=!1,wn=[],_n=null,xn=null,En=null,Sn=new Map,Tn=new Map,Pn=[],Cn="mousedown mouseup touchcancel touchend touchstart auxclick dblclick pointercancel pointerdown pointerup dragend dragstart drop compositionend compositionstart keydown keypress keyup input textInput close cancel copy cut paste click change contextmenu reset submit".split(" "),On="focus blur dragenter dragleave mouseover mouseout pointerover pointerout gotpointercapture lostpointercapture".split(" ");function An(e,n,t,i,r){return{blockedOn:e,topLevelType:n,eventSystemFlags:32|t,nativeEvent:r,container:i}}function Rn(e,n){switch(e){case"focus":case"blur":_n=null;break;case"dragenter":case"dragleave":xn=null;break;case"mouseover":case"mouseout":En=null;break;case"pointerover":case"pointerout":Sn.delete(n.pointerId);break;case"gotpointercapture":case"lostpointercapture":Tn.delete(n.pointerId)}}function Nn(e,n,t,i,r,o){return null===e||e.nativeEvent!==o?(e=An(n,t,i,r,o),null!==n&&(null!==(n=Nt(n))&&bn(n)),e):(e.eventSystemFlags|=i,e)}function In(e){var n=Rt(e.target);if(null!==n){var t=en(n);if(null!==t)if(13===(n=t.tag)){if(null!==(n=nn(t)))return e.blockedOn=n,void o.unstable_runWithPriority(e.priority,(function(){yn(t)}))}else if(3===n&&t.stateNode.hydrate)return void(e.blockedOn=3===t.tag?t.stateNode.containerInfo:null)}e.blockedOn=null}function Ln(e){if(null!==e.blockedOn)return!1;var n=et(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);if(null!==n){var t=Nt(n);return null!==t&&bn(t),e.blockedOn=n,!1}return!0}function Dn(e,n,t){Ln(e)&&t.delete(n)}function jn(){for(kn=!1;0<wn.length;){var e=wn[0];if(null!==e.blockedOn){null!==(e=Nt(e.blockedOn))&&vn(e);break}var n=et(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);null!==n?e.blockedOn=n:wn.shift()}null!==_n&&Ln(_n)&&(_n=null),null!==xn&&Ln(xn)&&(xn=null),null!==En&&Ln(En)&&(En=null),Sn.forEach(Dn),Tn.forEach(Dn)}function Mn(e,n){e.blockedOn===n&&(e.blockedOn=null,kn||(kn=!0,o.unstable_scheduleCallback(o.unstable_NormalPriority,jn)))}function Fn(e){function n(n){return Mn(n,e)}if(0<wn.length){Mn(wn[0],e);for(var t=1;t<wn.length;t++){var i=wn[t];i.blockedOn===e&&(i.blockedOn=null)}}for(null!==_n&&Mn(_n,e),null!==xn&&Mn(xn,e),null!==En&&Mn(En,e),Sn.forEach(n),Tn.forEach(n),t=0;t<Pn.length;t++)(i=Pn[t]).blockedOn===e&&(i.blockedOn=null);for(;0<Pn.length&&null===(t=Pn[0]).blockedOn;)In(t),null===t.blockedOn&&Pn.shift()}var Bn={},zn=new Map,Un=new Map,$n=["abort","abort",Ve,"animationEnd",Ke,"animationIteration",Ye,"animationStart","canplay","canPlay","canplaythrough","canPlayThrough","durationchange","durationChange","emptied","emptied","encrypted","encrypted","ended","ended","error","error","gotpointercapture","gotPointerCapture","load","load","loadeddata","loadedData","loadedmetadata","loadedMetadata","loadstart","loadStart","lostpointercapture","lostPointerCapture","playing","playing","progress","progress","seeking","seeking","stalled","stalled","suspend","suspend","timeupdate","timeUpdate",Qe,"transitionEnd","waiting","waiting"];function Gn(e,n){for(var t=0;t<e.length;t+=2){var i=e[t],r=e[t+1],o="on"+(r[0].toUpperCase()+r.slice(1));o={phasedRegistrationNames:{bubbled:o,captured:o+"Capture"},dependencies:[i],eventPriority:n},Un.set(i,n),zn.set(i,o),Bn[r]=o}}Gn("blur blur cancel cancel click click close close contextmenu contextMenu copy copy cut cut auxclick auxClick dblclick doubleClick dragend dragEnd dragstart dragStart drop drop focus focus input input invalid invalid keydown keyDown keypress keyPress keyup keyUp mousedown mouseDown mouseup mouseUp paste paste pause pause play play pointercancel pointerCancel pointerdown pointerDown pointerup pointerUp ratechange rateChange reset reset seeked seeked submit submit touchcancel touchCancel touchend touchEnd touchstart touchStart volumechange volumeChange".split(" "),0),Gn("drag drag dragenter dragEnter dragexit dragExit dragleave dragLeave dragover dragOver mousemove mouseMove mouseout mouseOut mouseover mouseOver pointermove pointerMove pointerout pointerOut pointerover pointerOver scroll scroll toggle toggle touchmove touchMove wheel wheel".split(" "),1),Gn($n,2);for(var qn="change selectionchange textInput compositionstart compositionend compositionupdate".split(" "),Hn=0;Hn<qn.length;Hn++)Un.set(qn[Hn],0);var Wn=o.unstable_UserBlockingPriority,Vn=o.unstable_runWithPriority,Kn=!0;function Yn(e,n){Qn(n,e,!1)}function Qn(e,n,t){var i=Un.get(n);switch(void 0===i?2:i){case 0:i=Xn.bind(null,n,1,e);break;case 1:i=Jn.bind(null,n,1,e);break;default:i=Zn.bind(null,n,1,e)}t?e.addEventListener(n,i,!0):e.addEventListener(n,i,!1)}function Xn(e,n,t,i){F||j();var r=Zn,o=F;F=!0;try{D(r,e,n,t,i)}finally{(F=o)||z()}}function Jn(e,n,t,i){Vn(Wn,Zn.bind(null,e,n,t,i))}function Zn(e,n,t,i){if(Kn)if(0<wn.length&&-1<Cn.indexOf(e))e=An(null,e,n,t,i),wn.push(e);else{var r=et(e,n,t,i);if(null===r)Rn(e,i);else if(-1<Cn.indexOf(e))e=An(r,e,n,t,i),wn.push(e);else if(!function(e,n,t,i,r){switch(n){case"focus":return _n=Nn(_n,e,n,t,i,r),!0;case"dragenter":return xn=Nn(xn,e,n,t,i,r),!0;case"mouseover":return En=Nn(En,e,n,t,i,r),!0;case"pointerover":var o=r.pointerId;return Sn.set(o,Nn(Sn.get(o)||null,e,n,t,i,r)),!0;case"gotpointercapture":return o=r.pointerId,Tn.set(o,Nn(Tn.get(o)||null,e,n,t,i,r)),!0}return!1}(r,e,n,t,i)){Rn(e,i),e=mn(e,i,null,n);try{U(gn,e)}finally{fn(e)}}}}function et(e,n,t,i){if(null!==(t=Rt(t=un(i)))){var r=en(t);if(null===r)t=null;else{var o=r.tag;if(13===o){if(null!==(t=nn(r)))return t;t=null}else if(3===o){if(r.stateNode.hydrate)return 3===r.tag?r.stateNode.containerInfo:null;t=null}else r!==t&&(t=null)}}e=mn(e,i,t,n);try{U(gn,e)}finally{fn(e)}return null}var nt={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},tt=["Webkit","ms","Moz","O"];function it(e,n,t){return null==n||"boolean"==typeof n||""===n?"":t||"number"!=typeof n||0===n||nt.hasOwnProperty(e)&&nt[e]?(""+n).trim():n+"px"}function rt(e,n){for(var t in e=e.style,n)if(n.hasOwnProperty(t)){var i=0===t.indexOf("--"),r=it(t,n[t],i);"float"===t&&(t="cssFloat"),i?e.setProperty(t,r):e[t]=r}}Object.keys(nt).forEach((function(e){tt.forEach((function(n){n=n+e.charAt(0).toUpperCase()+e.substring(1),nt[n]=nt[e]}))}));var ot=r({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function at(e,n){if(n){if(ot[e]&&(null!=n.children||null!=n.dangerouslySetInnerHTML))throw Error(a(137,e,""));if(null!=n.dangerouslySetInnerHTML){if(null!=n.children)throw Error(a(60));if("object"!=typeof n.dangerouslySetInnerHTML||!("__html"in n.dangerouslySetInnerHTML))throw Error(a(61))}if(null!=n.style&&"object"!=typeof n.style)throw Error(a(62,""))}}function lt(e,n){if(-1===e.indexOf("-"))return"string"==typeof n.is;switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}var st=Le;function ct(e,n){var t=Ze(e=9===e.nodeType||11===e.nodeType?e:e.ownerDocument);n=S[n];for(var i=0;i<n.length;i++)hn(n[i],e,t)}function ut(){}function pt(e){if(void 0===(e=e||("undefined"!=typeof document?document:void 0)))return null;try{return e.activeElement||e.body}catch(n){return e.body}}function dt(e){for(;e&&e.firstChild;)e=e.firstChild;return e}function ft(e,n){var t,i=dt(e);for(e=0;i;){if(3===i.nodeType){if(t=e+i.textContent.length,e<=n&&t>=n)return{node:i,offset:n-e};e=t}e:{for(;i;){if(i.nextSibling){i=i.nextSibling;break e}i=i.parentNode}i=void 0}i=dt(i)}}function mt(e,n){return!(!e||!n)&&(e===n||(!e||3!==e.nodeType)&&(n&&3===n.nodeType?mt(e,n.parentNode):"contains"in e?e.contains(n):!!e.compareDocumentPosition&&!!(16&e.compareDocumentPosition(n))))}function gt(){for(var e=window,n=pt();n instanceof e.HTMLIFrameElement;){try{var t="string"==typeof n.contentWindow.location.href}catch(i){t=!1}if(!t)break;n=pt((e=n.contentWindow).document)}return n}function ht(e){var n=e&&e.nodeName&&e.nodeName.toLowerCase();return n&&("input"===n&&("text"===e.type||"search"===e.type||"tel"===e.type||"url"===e.type||"password"===e.type)||"textarea"===n||"true"===e.contentEditable)}var vt="$?",bt="$!",yt=null,kt=null;function wt(e,n){switch(e){case"button":case"input":case"select":case"textarea":return!!n.autoFocus}return!1}function _t(e,n){return"textarea"===e||"option"===e||"noscript"===e||"string"==typeof n.children||"number"==typeof n.children||"object"==typeof n.dangerouslySetInnerHTML&&null!==n.dangerouslySetInnerHTML&&null!=n.dangerouslySetInnerHTML.__html}var xt="function"==typeof setTimeout?setTimeout:void 0,Et="function"==typeof clearTimeout?clearTimeout:void 0;function St(e){for(;null!=e;e=e.nextSibling){var n=e.nodeType;if(1===n||3===n)break}return e}function Tt(e){e=e.previousSibling;for(var n=0;e;){if(8===e.nodeType){var t=e.data;if("$"===t||t===bt||t===vt){if(0===n)return e;n--}else"/$"===t&&n++}e=e.previousSibling}return null}var Pt=Math.random().toString(36).slice(2),Ct="__reactInternalInstance$"+Pt,Ot="__reactEventHandlers$"+Pt,At="__reactContainere$"+Pt;function Rt(e){var n=e[Ct];if(n)return n;for(var t=e.parentNode;t;){if(n=t[At]||t[Ct]){if(t=n.alternate,null!==n.child||null!==t&&null!==t.child)for(e=Tt(e);null!==e;){if(t=e[Ct])return t;e=Tt(e)}return n}t=(e=t).parentNode}return null}function Nt(e){return!(e=e[Ct]||e[At])||5!==e.tag&&6!==e.tag&&13!==e.tag&&3!==e.tag?null:e}function It(e){if(5===e.tag||6===e.tag)return e.stateNode;throw Error(a(33))}function Lt(e){return e[Ot]||null}function Dt(e){do{e=e.return}while(e&&5!==e.tag);return e||null}function jt(e,n){var t=e.stateNode;if(!t)return null;var i=m(t);if(!i)return null;t=i[n];e:switch(n){case"onClick":case"onClickCapture":case"onDoubleClick":case"onDoubleClickCapture":case"onMouseDown":case"onMouseDownCapture":case"onMouseMove":case"onMouseMoveCapture":case"onMouseUp":case"onMouseUpCapture":case"onMouseEnter":(i=!i.disabled)||(i=!("button"===(e=e.type)||"input"===e||"select"===e||"textarea"===e)),e=!i;break e;default:e=!1}if(e)return null;if(t&&"function"!=typeof t)throw Error(a(231,n,typeof t));return t}function Mt(e,n,t){(n=jt(e,t.dispatchConfig.phasedRegistrationNames[n]))&&(t._dispatchListeners=on(t._dispatchListeners,n),t._dispatchInstances=on(t._dispatchInstances,e))}function Ft(e){if(e&&e.dispatchConfig.phasedRegistrationNames){for(var n=e._targetInst,t=[];n;)t.push(n),n=Dt(n);for(n=t.length;0<n--;)Mt(t[n],"captured",e);for(n=0;n<t.length;n++)Mt(t[n],"bubbled",e)}}function Bt(e,n,t){e&&t&&t.dispatchConfig.registrationName&&(n=jt(e,t.dispatchConfig.registrationName))&&(t._dispatchListeners=on(t._dispatchListeners,n),t._dispatchInstances=on(t._dispatchInstances,e))}function zt(e){e&&e.dispatchConfig.registrationName&&Bt(e._targetInst,null,e)}function Ut(e){an(e,Ft)}var $t=null,Gt=null,qt=null;function Ht(){if(qt)return qt;var e,n,t=Gt,i=t.length,r="value"in $t?$t.value:$t.textContent,o=r.length;for(e=0;e<i&&t[e]===r[e];e++);var a=i-e;for(n=1;n<=a&&t[i-n]===r[o-n];n++);return qt=r.slice(e,1<n?1-n:void 0)}function Wt(){return!0}function Vt(){return!1}function Kt(e,n,t,i){for(var r in this.dispatchConfig=e,this._targetInst=n,this.nativeEvent=t,e=this.constructor.Interface)e.hasOwnProperty(r)&&((n=e[r])?this[r]=n(t):"target"===r?this.target=i:this[r]=t[r]);return this.isDefaultPrevented=(null!=t.defaultPrevented?t.defaultPrevented:!1===t.returnValue)?Wt:Vt,this.isPropagationStopped=Vt,this}function Yt(e,n,t,i){if(this.eventPool.length){var r=this.eventPool.pop();return this.call(r,e,n,t,i),r}return new this(e,n,t,i)}function Qt(e){if(!(e instanceof this))throw Error(a(279));e.destructor(),10>this.eventPool.length&&this.eventPool.push(e)}function Xt(e){e.eventPool=[],e.getPooled=Yt,e.release=Qt}r(Kt.prototype,{preventDefault:function(){this.defaultPrevented=!0;var e=this.nativeEvent;e&&(e.preventDefault?e.preventDefault():"unknown"!=typeof e.returnValue&&(e.returnValue=!1),this.isDefaultPrevented=Wt)},stopPropagation:function(){var e=this.nativeEvent;e&&(e.stopPropagation?e.stopPropagation():"unknown"!=typeof e.cancelBubble&&(e.cancelBubble=!0),this.isPropagationStopped=Wt)},persist:function(){this.isPersistent=Wt},isPersistent:Vt,destructor:function(){var e,n=this.constructor.Interface;for(e in n)this[e]=null;this.nativeEvent=this._targetInst=this.dispatchConfig=null,this.isPropagationStopped=this.isDefaultPrevented=Vt,this._dispatchInstances=this._dispatchListeners=null}}),Kt.Interface={type:null,target:null,currentTarget:function(){return null},eventPhase:null,bubbles:null,cancelable:null,timeStamp:function(e){return e.timeStamp||Date.now()},defaultPrevented:null,isTrusted:null},Kt.extend=function(e){function n(){}function t(){return i.apply(this,arguments)}var i=this;n.prototype=i.prototype;var o=new n;return r(o,t.prototype),t.prototype=o,t.prototype.constructor=t,t.Interface=r({},i.Interface,e),t.extend=i.extend,Xt(t),t},Xt(Kt);var Jt=Kt.extend({data:null}),Zt=Kt.extend({data:null}),ei=[9,13,27,32],ni=P&&"CompositionEvent"in window,ti=null;P&&"documentMode"in document&&(ti=document.documentMode);var ii=P&&"TextEvent"in window&&!ti,ri=P&&(!ni||ti&&8<ti&&11>=ti),oi=String.fromCharCode(32),ai={beforeInput:{phasedRegistrationNames:{bubbled:"onBeforeInput",captured:"onBeforeInputCapture"},dependencies:["compositionend","keypress","textInput","paste"]},compositionEnd:{phasedRegistrationNames:{bubbled:"onCompositionEnd",captured:"onCompositionEndCapture"},dependencies:"blur compositionend keydown keypress keyup mousedown".split(" ")},compositionStart:{phasedRegistrationNames:{bubbled:"onCompositionStart",captured:"onCompositionStartCapture"},dependencies:"blur compositionstart keydown keypress keyup mousedown".split(" ")},compositionUpdate:{phasedRegistrationNames:{bubbled:"onCompositionUpdate",captured:"onCompositionUpdateCapture"},dependencies:"blur compositionupdate keydown keypress keyup mousedown".split(" ")}},li=!1;function si(e,n){switch(e){case"keyup":return-1!==ei.indexOf(n.keyCode);case"keydown":return 229!==n.keyCode;case"keypress":case"mousedown":case"blur":return!0;default:return!1}}function ci(e){return"object"==typeof(e=e.detail)&&"data"in e?e.data:null}var ui=!1;var pi={eventTypes:ai,extractEvents:function(e,n,t,i){var r;if(ni)e:{switch(e){case"compositionstart":var o=ai.compositionStart;break e;case"compositionend":o=ai.compositionEnd;break e;case"compositionupdate":o=ai.compositionUpdate;break e}o=void 0}else ui?si(e,t)&&(o=ai.compositionEnd):"keydown"===e&&229===t.keyCode&&(o=ai.compositionStart);return o?(ri&&"ko"!==t.locale&&(ui||o!==ai.compositionStart?o===ai.compositionEnd&&ui&&(r=Ht()):(Gt="value"in($t=i)?$t.value:$t.textContent,ui=!0)),o=Jt.getPooled(o,n,t,i),r?o.data=r:null!==(r=ci(t))&&(o.data=r),Ut(o),r=o):r=null,(e=ii?function(e,n){switch(e){case"compositionend":return ci(n);case"keypress":return 32!==n.which?null:(li=!0,oi);case"textInput":return(e=n.data)===oi&&li?null:e;default:return null}}(e,t):function(e,n){if(ui)return"compositionend"===e||!ni&&si(e,n)?(e=Ht(),qt=Gt=$t=null,ui=!1,e):null;switch(e){case"paste":return null;case"keypress":if(!(n.ctrlKey||n.altKey||n.metaKey)||n.ctrlKey&&n.altKey){if(n.char&&1<n.char.length)return n.char;if(n.which)return String.fromCharCode(n.which)}return null;case"compositionend":return ri&&"ko"!==n.locale?null:n.data;default:return null}}(e,t))?((n=Zt.getPooled(ai.beforeInput,n,t,i)).data=e,Ut(n)):n=null,null===r?n:null===n?r:[r,n]}},di={color:!0,date:!0,datetime:!0,"datetime-local":!0,email:!0,month:!0,number:!0,password:!0,range:!0,search:!0,tel:!0,text:!0,time:!0,url:!0,week:!0};function fi(e){var n=e&&e.nodeName&&e.nodeName.toLowerCase();return"input"===n?!!di[e.type]:"textarea"===n}var mi={change:{phasedRegistrationNames:{bubbled:"onChange",captured:"onChangeCapture"},dependencies:"blur change click focus input keydown keyup selectionchange".split(" ")}};function gi(e,n,t){return(e=Kt.getPooled(mi.change,e,n,t)).type="change",N(t),Ut(e),e}var hi=null,vi=null;function bi(e){cn(e)}function yi(e){if(we(It(e)))return e}function ki(e,n){if("change"===e)return n}var wi=!1;function _i(){hi&&(hi.detachEvent("onpropertychange",xi),vi=hi=null)}function xi(e){if("value"===e.propertyName&&yi(vi))if(e=gi(vi,e,un(e)),F)cn(e);else{F=!0;try{L(bi,e)}finally{F=!1,z()}}}function Ei(e,n,t){"focus"===e?(_i(),vi=t,(hi=n).attachEvent("onpropertychange",xi)):"blur"===e&&_i()}function Si(e){if("selectionchange"===e||"keyup"===e||"keydown"===e)return yi(vi)}function Ti(e,n){if("click"===e)return yi(n)}function Pi(e,n){if("input"===e||"change"===e)return yi(n)}P&&(wi=pn("input")&&(!document.documentMode||9<document.documentMode));var Ci={eventTypes:mi,_isInputEventSupported:wi,extractEvents:function(e,n,t,i){var r=n?It(n):window,o=r.nodeName&&r.nodeName.toLowerCase();if("select"===o||"input"===o&&"file"===r.type)var a=ki;else if(fi(r))if(wi)a=Pi;else{a=Si;var l=Ei}else(o=r.nodeName)&&"input"===o.toLowerCase()&&("checkbox"===r.type||"radio"===r.type)&&(a=Ti);if(a&&(a=a(e,n)))return gi(a,t,i);l&&l(e,r,n),"blur"===e&&(e=r._wrapperState)&&e.controlled&&"number"===r.type&&Pe(r,"number",r.value)}},Oi=Kt.extend({view:null,detail:null}),Ai={Alt:"altKey",Control:"ctrlKey",Meta:"metaKey",Shift:"shiftKey"};function Ri(e){var n=this.nativeEvent;return n.getModifierState?n.getModifierState(e):!!(e=Ai[e])&&!!n[e]}function Ni(){return Ri}var Ii=0,Li=0,Di=!1,ji=!1,Mi=Oi.extend({screenX:null,screenY:null,clientX:null,clientY:null,pageX:null,pageY:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,getModifierState:Ni,button:null,buttons:null,relatedTarget:function(e){return e.relatedTarget||(e.fromElement===e.srcElement?e.toElement:e.fromElement)},movementX:function(e){if("movementX"in e)return e.movementX;var n=Ii;return Ii=e.screenX,Di?"mousemove"===e.type?e.screenX-n:0:(Di=!0,0)},movementY:function(e){if("movementY"in e)return e.movementY;var n=Li;return Li=e.screenY,ji?"mousemove"===e.type?e.screenY-n:0:(ji=!0,0)}}),Fi=Mi.extend({pointerId:null,width:null,height:null,pressure:null,tangentialPressure:null,tiltX:null,tiltY:null,twist:null,pointerType:null,isPrimary:null}),Bi={mouseEnter:{registrationName:"onMouseEnter",dependencies:["mouseout","mouseover"]},mouseLeave:{registrationName:"onMouseLeave",dependencies:["mouseout","mouseover"]},pointerEnter:{registrationName:"onPointerEnter",dependencies:["pointerout","pointerover"]},pointerLeave:{registrationName:"onPointerLeave",dependencies:["pointerout","pointerover"]}},zi={eventTypes:Bi,extractEvents:function(e,n,t,i,r){var o="mouseover"===e||"pointerover"===e,a="mouseout"===e||"pointerout"===e;if(o&&0==(32&r)&&(t.relatedTarget||t.fromElement)||!a&&!o)return null;(o=i.window===i?i:(o=i.ownerDocument)?o.defaultView||o.parentWindow:window,a)?(a=n,null!==(n=(n=t.relatedTarget||t.toElement)?Rt(n):null)&&(n!==en(n)||5!==n.tag&&6!==n.tag)&&(n=null)):a=null;if(a===n)return null;if("mouseout"===e||"mouseover"===e)var l=Mi,s=Bi.mouseLeave,c=Bi.mouseEnter,u="mouse";else"pointerout"!==e&&"pointerover"!==e||(l=Fi,s=Bi.pointerLeave,c=Bi.pointerEnter,u="pointer");if(e=null==a?o:It(a),o=null==n?o:It(n),(s=l.getPooled(s,a,t,i)).type=u+"leave",s.target=e,s.relatedTarget=o,(t=l.getPooled(c,n,t,i)).type=u+"enter",t.target=o,t.relatedTarget=e,u=n,(i=a)&&u)e:{for(c=u,a=0,e=l=i;e;e=Dt(e))a++;for(e=0,n=c;n;n=Dt(n))e++;for(;0<a-e;)l=Dt(l),a--;for(;0<e-a;)c=Dt(c),e--;for(;a--;){if(l===c||l===c.alternate)break e;l=Dt(l),c=Dt(c)}l=null}else l=null;for(c=l,l=[];i&&i!==c&&(null===(a=i.alternate)||a!==c);)l.push(i),i=Dt(i);for(i=[];u&&u!==c&&(null===(a=u.alternate)||a!==c);)i.push(u),u=Dt(u);for(u=0;u<l.length;u++)Bt(l[u],"bubbled",s);for(u=i.length;0<u--;)Bt(i[u],"captured",t);return 0==(64&r)?[s]:[s,t]}};var Ui="function"==typeof Object.is?Object.is:function(e,n){return e===n&&(0!==e||1/e==1/n)||e!=e&&n!=n},$i=Object.prototype.hasOwnProperty;function Gi(e,n){if(Ui(e,n))return!0;if("object"!=typeof e||null===e||"object"!=typeof n||null===n)return!1;var t=Object.keys(e),i=Object.keys(n);if(t.length!==i.length)return!1;for(i=0;i<t.length;i++)if(!$i.call(n,t[i])||!Ui(e[t[i]],n[t[i]]))return!1;return!0}var qi=P&&"documentMode"in document&&11>=document.documentMode,Hi={select:{phasedRegistrationNames:{bubbled:"onSelect",captured:"onSelectCapture"},dependencies:"blur contextmenu dragend focus keydown keyup mousedown mouseup selectionchange".split(" ")}},Wi=null,Vi=null,Ki=null,Yi=!1;function Qi(e,n){var t=n.window===n?n.document:9===n.nodeType?n:n.ownerDocument;return Yi||null==Wi||Wi!==pt(t)?null:("selectionStart"in(t=Wi)&&ht(t)?t={start:t.selectionStart,end:t.selectionEnd}:t={anchorNode:(t=(t.ownerDocument&&t.ownerDocument.defaultView||window).getSelection()).anchorNode,anchorOffset:t.anchorOffset,focusNode:t.focusNode,focusOffset:t.focusOffset},Ki&&Gi(Ki,t)?null:(Ki=t,(e=Kt.getPooled(Hi.select,Vi,e,n)).type="select",e.target=Wi,Ut(e),e))}var Xi={eventTypes:Hi,extractEvents:function(e,n,t,i,r,o){if(!(o=!(r=o||(i.window===i?i.document:9===i.nodeType?i:i.ownerDocument)))){e:{r=Ze(r),o=S.onSelect;for(var a=0;a<o.length;a++)if(!r.has(o[a])){r=!1;break e}r=!0}o=!r}if(o)return null;switch(r=n?It(n):window,e){case"focus":(fi(r)||"true"===r.contentEditable)&&(Wi=r,Vi=n,Ki=null);break;case"blur":Ki=Vi=Wi=null;break;case"mousedown":Yi=!0;break;case"contextmenu":case"mouseup":case"dragend":return Yi=!1,Qi(t,i);case"selectionchange":if(qi)break;case"keydown":case"keyup":return Qi(t,i)}return null}},Ji=Kt.extend({animationName:null,elapsedTime:null,pseudoElement:null}),Zi=Kt.extend({clipboardData:function(e){return"clipboardData"in e?e.clipboardData:window.clipboardData}}),er=Oi.extend({relatedTarget:null});function nr(e){var n=e.keyCode;return"charCode"in e?0===(e=e.charCode)&&13===n&&(e=13):e=n,10===e&&(e=13),32<=e||13===e?e:0}var tr={Esc:"Escape",Spacebar:" ",Left:"ArrowLeft",Up:"ArrowUp",Right:"ArrowRight",Down:"ArrowDown",Del:"Delete",Win:"OS",Menu:"ContextMenu",Apps:"ContextMenu",Scroll:"ScrollLock",MozPrintableKey:"Unidentified"},ir={8:"Backspace",9:"Tab",12:"Clear",13:"Enter",16:"Shift",17:"Control",18:"Alt",19:"Pause",20:"CapsLock",27:"Escape",32:" ",33:"PageUp",34:"PageDown",35:"End",36:"Home",37:"ArrowLeft",38:"ArrowUp",39:"ArrowRight",40:"ArrowDown",45:"Insert",46:"Delete",112:"F1",113:"F2",114:"F3",115:"F4",116:"F5",117:"F6",118:"F7",119:"F8",120:"F9",121:"F10",122:"F11",123:"F12",144:"NumLock",145:"ScrollLock",224:"Meta"},rr=Oi.extend({key:function(e){if(e.key){var n=tr[e.key]||e.key;if("Unidentified"!==n)return n}return"keypress"===e.type?13===(e=nr(e))?"Enter":String.fromCharCode(e):"keydown"===e.type||"keyup"===e.type?ir[e.keyCode]||"Unidentified":""},location:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,repeat:null,locale:null,getModifierState:Ni,charCode:function(e){return"keypress"===e.type?nr(e):0},keyCode:function(e){return"keydown"===e.type||"keyup"===e.type?e.keyCode:0},which:function(e){return"keypress"===e.type?nr(e):"keydown"===e.type||"keyup"===e.type?e.keyCode:0}}),or=Mi.extend({dataTransfer:null}),ar=Oi.extend({touches:null,targetTouches:null,changedTouches:null,altKey:null,metaKey:null,ctrlKey:null,shiftKey:null,getModifierState:Ni}),lr=Kt.extend({propertyName:null,elapsedTime:null,pseudoElement:null}),sr=Mi.extend({deltaX:function(e){return"deltaX"in e?e.deltaX:"wheelDeltaX"in e?-e.wheelDeltaX:0},deltaY:function(e){return"deltaY"in e?e.deltaY:"wheelDeltaY"in e?-e.wheelDeltaY:"wheelDelta"in e?-e.wheelDelta:0},deltaZ:null,deltaMode:null}),cr={eventTypes:Bn,extractEvents:function(e,n,t,i){var r=zn.get(e);if(!r)return null;switch(e){case"keypress":if(0===nr(t))return null;case"keydown":case"keyup":e=rr;break;case"blur":case"focus":e=er;break;case"click":if(2===t.button)return null;case"auxclick":case"dblclick":case"mousedown":case"mousemove":case"mouseup":case"mouseout":case"mouseover":case"contextmenu":e=Mi;break;case"drag":case"dragend":case"dragenter":case"dragexit":case"dragleave":case"dragover":case"dragstart":case"drop":e=or;break;case"touchcancel":case"touchend":case"touchmove":case"touchstart":e=ar;break;case Ve:case Ke:case Ye:e=Ji;break;case Qe:e=lr;break;case"scroll":e=Oi;break;case"wheel":e=sr;break;case"copy":case"cut":case"paste":e=Zi;break;case"gotpointercapture":case"lostpointercapture":case"pointercancel":case"pointerdown":case"pointermove":case"pointerout":case"pointerover":case"pointerup":e=Fi;break;default:e=Kt}return Ut(n=e.getPooled(r,n,t,i)),n}};if(b)throw Error(a(101));b=Array.prototype.slice.call("ResponderEventPlugin SimpleEventPlugin EnterLeaveEventPlugin ChangeEventPlugin SelectEventPlugin BeforeInputEventPlugin".split(" ")),k(),m=Lt,g=Nt,h=It,T({SimpleEventPlugin:cr,EnterLeaveEventPlugin:zi,ChangeEventPlugin:Ci,SelectEventPlugin:Xi,BeforeInputEventPlugin:pi});var ur=[],pr=-1;function dr(e){0>pr||(e.current=ur[pr],ur[pr]=null,pr--)}function fr(e,n){pr++,ur[pr]=e.current,e.current=n}var mr={},gr={current:mr},hr={current:!1},vr=mr;function br(e,n){var t=e.type.contextTypes;if(!t)return mr;var i=e.stateNode;if(i&&i.__reactInternalMemoizedUnmaskedChildContext===n)return i.__reactInternalMemoizedMaskedChildContext;var r,o={};for(r in t)o[r]=n[r];return i&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=n,e.__reactInternalMemoizedMaskedChildContext=o),o}function yr(e){return null!=(e=e.childContextTypes)}function kr(){dr(hr),dr(gr)}function wr(e,n,t){if(gr.current!==mr)throw Error(a(168));fr(gr,n),fr(hr,t)}function _r(e,n,t){var i=e.stateNode;if(e=n.childContextTypes,"function"!=typeof i.getChildContext)return t;for(var o in i=i.getChildContext())if(!(o in e))throw Error(a(108,he(n)||"Unknown",o));return r({},t,{},i)}function xr(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||mr,vr=gr.current,fr(gr,e),fr(hr,hr.current),!0}function Er(e,n,t){var i=e.stateNode;if(!i)throw Error(a(169));t?(e=_r(e,n,vr),i.__reactInternalMemoizedMergedChildContext=e,dr(hr),dr(gr),fr(gr,e)):dr(hr),fr(hr,t)}var Sr=o.unstable_runWithPriority,Tr=o.unstable_scheduleCallback,Pr=o.unstable_cancelCallback,Cr=o.unstable_requestPaint,Or=o.unstable_now,Ar=o.unstable_getCurrentPriorityLevel,Rr=o.unstable_ImmediatePriority,Nr=o.unstable_UserBlockingPriority,Ir=o.unstable_NormalPriority,Lr=o.unstable_LowPriority,Dr=o.unstable_IdlePriority,jr={},Mr=o.unstable_shouldYield,Fr=void 0!==Cr?Cr:function(){},Br=null,zr=null,Ur=!1,$r=Or(),Gr=1e4>$r?Or:function(){return Or()-$r};function qr(){switch(Ar()){case Rr:return 99;case Nr:return 98;case Ir:return 97;case Lr:return 96;case Dr:return 95;default:throw Error(a(332))}}function Hr(e){switch(e){case 99:return Rr;case 98:return Nr;case 97:return Ir;case 96:return Lr;case 95:return Dr;default:throw Error(a(332))}}function Wr(e,n){return e=Hr(e),Sr(e,n)}function Vr(e,n,t){return e=Hr(e),Tr(e,n,t)}function Kr(e){return null===Br?(Br=[e],zr=Tr(Rr,Qr)):Br.push(e),jr}function Yr(){if(null!==zr){var e=zr;zr=null,Pr(e)}Qr()}function Qr(){if(!Ur&&null!==Br){Ur=!0;var e=0;try{var n=Br;Wr(99,(function(){for(;e<n.length;e++){var t=n[e];do{t=t(!0)}while(null!==t)}})),Br=null}catch(t){throw null!==Br&&(Br=Br.slice(e+1)),Tr(Rr,Yr),t}finally{Ur=!1}}}function Xr(e,n,t){return 1073741821-(1+((1073741821-e+n/10)/(t/=10)|0))*t}function Jr(e,n){if(e&&e.defaultProps)for(var t in n=r({},n),e=e.defaultProps)void 0===n[t]&&(n[t]=e[t]);return n}var Zr={current:null},eo=null,no=null,to=null;function io(){to=no=eo=null}function ro(e){var n=Zr.current;dr(Zr),e.type._context._currentValue=n}function oo(e,n){for(;null!==e;){var t=e.alternate;if(e.childExpirationTime<n)e.childExpirationTime=n,null!==t&&t.childExpirationTime<n&&(t.childExpirationTime=n);else{if(!(null!==t&&t.childExpirationTime<n))break;t.childExpirationTime=n}e=e.return}}function ao(e,n){eo=e,to=no=null,null!==(e=e.dependencies)&&null!==e.firstContext&&(e.expirationTime>=n&&(La=!0),e.firstContext=null)}function lo(e,n){if(to!==e&&!1!==n&&0!==n)if("number"==typeof n&&1073741823!==n||(to=e,n=1073741823),n={context:e,observedBits:n,next:null},null===no){if(null===eo)throw Error(a(308));no=n,eo.dependencies={expirationTime:0,firstContext:n,responders:null}}else no=no.next=n;return e._currentValue}var so=!1;function co(e){e.updateQueue={baseState:e.memoizedState,baseQueue:null,shared:{pending:null},effects:null}}function uo(e,n){e=e.updateQueue,n.updateQueue===e&&(n.updateQueue={baseState:e.baseState,baseQueue:e.baseQueue,shared:e.shared,effects:e.effects})}function po(e,n){return(e={expirationTime:e,suspenseConfig:n,tag:0,payload:null,callback:null,next:null}).next=e}function fo(e,n){if(null!==(e=e.updateQueue)){var t=(e=e.shared).pending;null===t?n.next=n:(n.next=t.next,t.next=n),e.pending=n}}function mo(e,n){var t=e.alternate;null!==t&&uo(t,e),null===(t=(e=e.updateQueue).baseQueue)?(e.baseQueue=n.next=n,n.next=n):(n.next=t.next,t.next=n)}function go(e,n,t,i){var o=e.updateQueue;so=!1;var a=o.baseQueue,l=o.shared.pending;if(null!==l){if(null!==a){var s=a.next;a.next=l.next,l.next=s}a=l,o.shared.pending=null,null!==(s=e.alternate)&&(null!==(s=s.updateQueue)&&(s.baseQueue=l))}if(null!==a){s=a.next;var c=o.baseState,u=0,p=null,d=null,f=null;if(null!==s)for(var m=s;;){if((l=m.expirationTime)<i){var g={expirationTime:m.expirationTime,suspenseConfig:m.suspenseConfig,tag:m.tag,payload:m.payload,callback:m.callback,next:null};null===f?(d=f=g,p=c):f=f.next=g,l>u&&(u=l)}else{null!==f&&(f=f.next={expirationTime:1073741823,suspenseConfig:m.suspenseConfig,tag:m.tag,payload:m.payload,callback:m.callback,next:null}),fs(l,m.suspenseConfig);e:{var h=e,v=m;switch(l=n,g=t,v.tag){case 1:if("function"==typeof(h=v.payload)){c=h.call(g,c,l);break e}c=h;break e;case 3:h.effectTag=-4097&h.effectTag|64;case 0:if(null==(l="function"==typeof(h=v.payload)?h.call(g,c,l):h))break e;c=r({},c,l);break e;case 2:so=!0}}null!==m.callback&&(e.effectTag|=32,null===(l=o.effects)?o.effects=[m]:l.push(m))}if(null===(m=m.next)||m===s){if(null===(l=o.shared.pending))break;m=a.next=l.next,l.next=s,o.baseQueue=a=l,o.shared.pending=null}}null===f?p=c:f.next=d,o.baseState=p,o.baseQueue=f,ms(u),e.expirationTime=u,e.memoizedState=c}}function ho(e,n,t){if(e=n.effects,n.effects=null,null!==e)for(n=0;n<e.length;n++){var i=e[n],r=i.callback;if(null!==r){if(i.callback=null,i=r,r=t,"function"!=typeof i)throw Error(a(191,i));i.call(r)}}}var vo=Q.ReactCurrentBatchConfig,bo=(new i.Component).refs;function yo(e,n,t,i){t=null==(t=t(i,n=e.memoizedState))?n:r({},n,t),e.memoizedState=t,0===e.expirationTime&&(e.updateQueue.baseState=t)}var ko={isMounted:function(e){return!!(e=e._reactInternalFiber)&&en(e)===e},enqueueSetState:function(e,n,t){e=e._reactInternalFiber;var i=es(),r=vo.suspense;(r=po(i=ns(i,e,r),r)).payload=n,null!=t&&(r.callback=t),fo(e,r),ts(e,i)},enqueueReplaceState:function(e,n,t){e=e._reactInternalFiber;var i=es(),r=vo.suspense;(r=po(i=ns(i,e,r),r)).tag=1,r.payload=n,null!=t&&(r.callback=t),fo(e,r),ts(e,i)},enqueueForceUpdate:function(e,n){e=e._reactInternalFiber;var t=es(),i=vo.suspense;(i=po(t=ns(t,e,i),i)).tag=2,null!=n&&(i.callback=n),fo(e,i),ts(e,t)}};function wo(e,n,t,i,r,o,a){return"function"==typeof(e=e.stateNode).shouldComponentUpdate?e.shouldComponentUpdate(i,o,a):!n.prototype||!n.prototype.isPureReactComponent||(!Gi(t,i)||!Gi(r,o))}function _o(e,n,t){var i=!1,r=mr,o=n.contextType;return"object"==typeof o&&null!==o?o=lo(o):(r=yr(n)?vr:gr.current,o=(i=null!=(i=n.contextTypes))?br(e,r):mr),n=new n(t,o),e.memoizedState=null!==n.state&&void 0!==n.state?n.state:null,n.updater=ko,e.stateNode=n,n._reactInternalFiber=e,i&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=r,e.__reactInternalMemoizedMaskedChildContext=o),n}function xo(e,n,t,i){e=n.state,"function"==typeof n.componentWillReceiveProps&&n.componentWillReceiveProps(t,i),"function"==typeof n.UNSAFE_componentWillReceiveProps&&n.UNSAFE_componentWillReceiveProps(t,i),n.state!==e&&ko.enqueueReplaceState(n,n.state,null)}function Eo(e,n,t,i){var r=e.stateNode;r.props=t,r.state=e.memoizedState,r.refs=bo,co(e);var o=n.contextType;"object"==typeof o&&null!==o?r.context=lo(o):(o=yr(n)?vr:gr.current,r.context=br(e,o)),go(e,t,r,i),r.state=e.memoizedState,"function"==typeof(o=n.getDerivedStateFromProps)&&(yo(e,n,o,t),r.state=e.memoizedState),"function"==typeof n.getDerivedStateFromProps||"function"==typeof r.getSnapshotBeforeUpdate||"function"!=typeof r.UNSAFE_componentWillMount&&"function"!=typeof r.componentWillMount||(n=r.state,"function"==typeof r.componentWillMount&&r.componentWillMount(),"function"==typeof r.UNSAFE_componentWillMount&&r.UNSAFE_componentWillMount(),n!==r.state&&ko.enqueueReplaceState(r,r.state,null),go(e,t,r,i),r.state=e.memoizedState),"function"==typeof r.componentDidMount&&(e.effectTag|=4)}var So=Array.isArray;function To(e,n,t){if(null!==(e=t.ref)&&"function"!=typeof e&&"object"!=typeof e){if(t._owner){if(t=t._owner){if(1!==t.tag)throw Error(a(309));var i=t.stateNode}if(!i)throw Error(a(147,e));var r=""+e;return null!==n&&null!==n.ref&&"function"==typeof n.ref&&n.ref._stringRef===r?n.ref:((n=function(e){var n=i.refs;n===bo&&(n=i.refs={}),null===e?delete n[r]:n[r]=e})._stringRef=r,n)}if("string"!=typeof e)throw Error(a(284));if(!t._owner)throw Error(a(290,e))}return e}function Po(e,n){if("textarea"!==e.type)throw Error(a(31,"[object Object]"===Object.prototype.toString.call(n)?"object with keys {"+Object.keys(n).join(", ")+"}":n,""))}function Co(e){function n(n,t){if(e){var i=n.lastEffect;null!==i?(i.nextEffect=t,n.lastEffect=t):n.firstEffect=n.lastEffect=t,t.nextEffect=null,t.effectTag=8}}function t(t,i){if(!e)return null;for(;null!==i;)n(t,i),i=i.sibling;return null}function i(e,n){for(e=new Map;null!==n;)null!==n.key?e.set(n.key,n):e.set(n.index,n),n=n.sibling;return e}function r(e,n){return(e=Ls(e,n)).index=0,e.sibling=null,e}function o(n,t,i){return n.index=i,e?null!==(i=n.alternate)?(i=i.index)<t?(n.effectTag=2,t):i:(n.effectTag=2,t):t}function l(n){return e&&null===n.alternate&&(n.effectTag=2),n}function s(e,n,t,i){return null===n||6!==n.tag?((n=Ms(t,e.mode,i)).return=e,n):((n=r(n,t)).return=e,n)}function c(e,n,t,i){return null!==n&&n.elementType===t.type?((i=r(n,t.props)).ref=To(e,n,t),i.return=e,i):((i=Ds(t.type,t.key,t.props,null,e.mode,i)).ref=To(e,n,t),i.return=e,i)}function u(e,n,t,i){return null===n||4!==n.tag||n.stateNode.containerInfo!==t.containerInfo||n.stateNode.implementation!==t.implementation?((n=Fs(t,e.mode,i)).return=e,n):((n=r(n,t.children||[])).return=e,n)}function p(e,n,t,i,o){return null===n||7!==n.tag?((n=js(t,e.mode,i,o)).return=e,n):((n=r(n,t)).return=e,n)}function d(e,n,t){if("string"==typeof n||"number"==typeof n)return(n=Ms(""+n,e.mode,t)).return=e,n;if("object"==typeof n&&null!==n){switch(n.$$typeof){case ee:return(t=Ds(n.type,n.key,n.props,null,e.mode,t)).ref=To(e,null,n),t.return=e,t;case ne:return(n=Fs(n,e.mode,t)).return=e,n}if(So(n)||ge(n))return(n=js(n,e.mode,t,null)).return=e,n;Po(e,n)}return null}function f(e,n,t,i){var r=null!==n?n.key:null;if("string"==typeof t||"number"==typeof t)return null!==r?null:s(e,n,""+t,i);if("object"==typeof t&&null!==t){switch(t.$$typeof){case ee:return t.key===r?t.type===te?p(e,n,t.props.children,i,r):c(e,n,t,i):null;case ne:return t.key===r?u(e,n,t,i):null}if(So(t)||ge(t))return null!==r?null:p(e,n,t,i,null);Po(e,t)}return null}function m(e,n,t,i,r){if("string"==typeof i||"number"==typeof i)return s(n,e=e.get(t)||null,""+i,r);if("object"==typeof i&&null!==i){switch(i.$$typeof){case ee:return e=e.get(null===i.key?t:i.key)||null,i.type===te?p(n,e,i.props.children,r,i.key):c(n,e,i,r);case ne:return u(n,e=e.get(null===i.key?t:i.key)||null,i,r)}if(So(i)||ge(i))return p(n,e=e.get(t)||null,i,r,null);Po(n,i)}return null}function g(r,a,l,s){for(var c=null,u=null,p=a,g=a=0,h=null;null!==p&&g<l.length;g++){p.index>g?(h=p,p=null):h=p.sibling;var v=f(r,p,l[g],s);if(null===v){null===p&&(p=h);break}e&&p&&null===v.alternate&&n(r,p),a=o(v,a,g),null===u?c=v:u.sibling=v,u=v,p=h}if(g===l.length)return t(r,p),c;if(null===p){for(;g<l.length;g++)null!==(p=d(r,l[g],s))&&(a=o(p,a,g),null===u?c=p:u.sibling=p,u=p);return c}for(p=i(r,p);g<l.length;g++)null!==(h=m(p,r,g,l[g],s))&&(e&&null!==h.alternate&&p.delete(null===h.key?g:h.key),a=o(h,a,g),null===u?c=h:u.sibling=h,u=h);return e&&p.forEach((function(e){return n(r,e)})),c}function h(r,l,s,c){var u=ge(s);if("function"!=typeof u)throw Error(a(150));if(null==(s=u.call(s)))throw Error(a(151));for(var p=u=null,g=l,h=l=0,v=null,b=s.next();null!==g&&!b.done;h++,b=s.next()){g.index>h?(v=g,g=null):v=g.sibling;var y=f(r,g,b.value,c);if(null===y){null===g&&(g=v);break}e&&g&&null===y.alternate&&n(r,g),l=o(y,l,h),null===p?u=y:p.sibling=y,p=y,g=v}if(b.done)return t(r,g),u;if(null===g){for(;!b.done;h++,b=s.next())null!==(b=d(r,b.value,c))&&(l=o(b,l,h),null===p?u=b:p.sibling=b,p=b);return u}for(g=i(r,g);!b.done;h++,b=s.next())null!==(b=m(g,r,h,b.value,c))&&(e&&null!==b.alternate&&g.delete(null===b.key?h:b.key),l=o(b,l,h),null===p?u=b:p.sibling=b,p=b);return e&&g.forEach((function(e){return n(r,e)})),u}return function(e,i,o,s){var c="object"==typeof o&&null!==o&&o.type===te&&null===o.key;c&&(o=o.props.children);var u="object"==typeof o&&null!==o;if(u)switch(o.$$typeof){case ee:e:{for(u=o.key,c=i;null!==c;){if(c.key===u){switch(c.tag){case 7:if(o.type===te){t(e,c.sibling),(i=r(c,o.props.children)).return=e,e=i;break e}break;default:if(c.elementType===o.type){t(e,c.sibling),(i=r(c,o.props)).ref=To(e,c,o),i.return=e,e=i;break e}}t(e,c);break}n(e,c),c=c.sibling}o.type===te?((i=js(o.props.children,e.mode,s,o.key)).return=e,e=i):((s=Ds(o.type,o.key,o.props,null,e.mode,s)).ref=To(e,i,o),s.return=e,e=s)}return l(e);case ne:e:{for(c=o.key;null!==i;){if(i.key===c){if(4===i.tag&&i.stateNode.containerInfo===o.containerInfo&&i.stateNode.implementation===o.implementation){t(e,i.sibling),(i=r(i,o.children||[])).return=e,e=i;break e}t(e,i);break}n(e,i),i=i.sibling}(i=Fs(o,e.mode,s)).return=e,e=i}return l(e)}if("string"==typeof o||"number"==typeof o)return o=""+o,null!==i&&6===i.tag?(t(e,i.sibling),(i=r(i,o)).return=e,e=i):(t(e,i),(i=Ms(o,e.mode,s)).return=e,e=i),l(e);if(So(o))return g(e,i,o,s);if(ge(o))return h(e,i,o,s);if(u&&Po(e,o),void 0===o&&!c)switch(e.tag){case 1:case 0:throw e=e.type,Error(a(152,e.displayName||e.name||"Component"))}return t(e,i)}}var Oo=Co(!0),Ao=Co(!1),Ro={},No={current:Ro},Io={current:Ro},Lo={current:Ro};function Do(e){if(e===Ro)throw Error(a(174));return e}function jo(e,n){switch(fr(Lo,n),fr(Io,e),fr(No,Ro),e=n.nodeType){case 9:case 11:n=(n=n.documentElement)?n.namespaceURI:Me(null,"");break;default:n=Me(n=(e=8===e?n.parentNode:n).namespaceURI||null,e=e.tagName)}dr(No),fr(No,n)}function Mo(){dr(No),dr(Io),dr(Lo)}function Fo(e){Do(Lo.current);var n=Do(No.current),t=Me(n,e.type);n!==t&&(fr(Io,e),fr(No,t))}function Bo(e){Io.current===e&&(dr(No),dr(Io))}var zo={current:0};function Uo(e){for(var n=e;null!==n;){if(13===n.tag){var t=n.memoizedState;if(null!==t&&(null===(t=t.dehydrated)||t.data===vt||t.data===bt))return n}else if(19===n.tag&&void 0!==n.memoizedProps.revealOrder){if(0!=(64&n.effectTag))return n}else if(null!==n.child){n.child.return=n,n=n.child;continue}if(n===e)break;for(;null===n.sibling;){if(null===n.return||n.return===e)return null;n=n.return}n.sibling.return=n.return,n=n.sibling}return null}function $o(e,n){return{responder:e,props:n}}var Go=Q.ReactCurrentDispatcher,qo=Q.ReactCurrentBatchConfig,Ho=0,Wo=null,Vo=null,Ko=null,Yo=!1;function Qo(){throw Error(a(321))}function Xo(e,n){if(null===n)return!1;for(var t=0;t<n.length&&t<e.length;t++)if(!Ui(e[t],n[t]))return!1;return!0}function Jo(e,n,t,i,r,o){if(Ho=o,Wo=n,n.memoizedState=null,n.updateQueue=null,n.expirationTime=0,Go.current=null===e||null===e.memoizedState?wa:_a,e=t(i,r),n.expirationTime===Ho){o=0;do{if(n.expirationTime=0,!(25>o))throw Error(a(301));o+=1,Ko=Vo=null,n.updateQueue=null,Go.current=xa,e=t(i,r)}while(n.expirationTime===Ho)}if(Go.current=ka,n=null!==Vo&&null!==Vo.next,Ho=0,Ko=Vo=Wo=null,Yo=!1,n)throw Error(a(300));return e}function Zo(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return null===Ko?Wo.memoizedState=Ko=e:Ko=Ko.next=e,Ko}function ea(){if(null===Vo){var e=Wo.alternate;e=null!==e?e.memoizedState:null}else e=Vo.next;var n=null===Ko?Wo.memoizedState:Ko.next;if(null!==n)Ko=n,Vo=e;else{if(null===e)throw Error(a(310));e={memoizedState:(Vo=e).memoizedState,baseState:Vo.baseState,baseQueue:Vo.baseQueue,queue:Vo.queue,next:null},null===Ko?Wo.memoizedState=Ko=e:Ko=Ko.next=e}return Ko}function na(e,n){return"function"==typeof n?n(e):n}function ta(e){var n=ea(),t=n.queue;if(null===t)throw Error(a(311));t.lastRenderedReducer=e;var i=Vo,r=i.baseQueue,o=t.pending;if(null!==o){if(null!==r){var l=r.next;r.next=o.next,o.next=l}i.baseQueue=r=o,t.pending=null}if(null!==r){r=r.next,i=i.baseState;var s=l=o=null,c=r;do{var u=c.expirationTime;if(u<Ho){var p={expirationTime:c.expirationTime,suspenseConfig:c.suspenseConfig,action:c.action,eagerReducer:c.eagerReducer,eagerState:c.eagerState,next:null};null===s?(l=s=p,o=i):s=s.next=p,u>Wo.expirationTime&&(Wo.expirationTime=u,ms(u))}else null!==s&&(s=s.next={expirationTime:1073741823,suspenseConfig:c.suspenseConfig,action:c.action,eagerReducer:c.eagerReducer,eagerState:c.eagerState,next:null}),fs(u,c.suspenseConfig),i=c.eagerReducer===e?c.eagerState:e(i,c.action);c=c.next}while(null!==c&&c!==r);null===s?o=i:s.next=l,Ui(i,n.memoizedState)||(La=!0),n.memoizedState=i,n.baseState=o,n.baseQueue=s,t.lastRenderedState=i}return[n.memoizedState,t.dispatch]}function ia(e){var n=ea(),t=n.queue;if(null===t)throw Error(a(311));t.lastRenderedReducer=e;var i=t.dispatch,r=t.pending,o=n.memoizedState;if(null!==r){t.pending=null;var l=r=r.next;do{o=e(o,l.action),l=l.next}while(l!==r);Ui(o,n.memoizedState)||(La=!0),n.memoizedState=o,null===n.baseQueue&&(n.baseState=o),t.lastRenderedState=o}return[o,i]}function ra(e){var n=Zo();return"function"==typeof e&&(e=e()),n.memoizedState=n.baseState=e,e=(e=n.queue={pending:null,dispatch:null,lastRenderedReducer:na,lastRenderedState:e}).dispatch=ya.bind(null,Wo,e),[n.memoizedState,e]}function oa(e,n,t,i){return e={tag:e,create:n,destroy:t,deps:i,next:null},null===(n=Wo.updateQueue)?(n={lastEffect:null},Wo.updateQueue=n,n.lastEffect=e.next=e):null===(t=n.lastEffect)?n.lastEffect=e.next=e:(i=t.next,t.next=e,e.next=i,n.lastEffect=e),e}function aa(){return ea().memoizedState}function la(e,n,t,i){var r=Zo();Wo.effectTag|=e,r.memoizedState=oa(1|n,t,void 0,void 0===i?null:i)}function sa(e,n,t,i){var r=ea();i=void 0===i?null:i;var o=void 0;if(null!==Vo){var a=Vo.memoizedState;if(o=a.destroy,null!==i&&Xo(i,a.deps))return void oa(n,t,o,i)}Wo.effectTag|=e,r.memoizedState=oa(1|n,t,o,i)}function ca(e,n){return la(516,4,e,n)}function ua(e,n){return sa(516,4,e,n)}function pa(e,n){return sa(4,2,e,n)}function da(e,n){return"function"==typeof n?(e=e(),n(e),function(){n(null)}):null!=n?(e=e(),n.current=e,function(){n.current=null}):void 0}function fa(e,n,t){return t=null!=t?t.concat([e]):null,sa(4,2,da.bind(null,n,e),t)}function ma(){}function ga(e,n){return Zo().memoizedState=[e,void 0===n?null:n],e}function ha(e,n){var t=ea();n=void 0===n?null:n;var i=t.memoizedState;return null!==i&&null!==n&&Xo(n,i[1])?i[0]:(t.memoizedState=[e,n],e)}function va(e,n){var t=ea();n=void 0===n?null:n;var i=t.memoizedState;return null!==i&&null!==n&&Xo(n,i[1])?i[0]:(e=e(),t.memoizedState=[e,n],e)}function ba(e,n,t){var i=qr();Wr(98>i?98:i,(function(){e(!0)})),Wr(97<i?97:i,(function(){var i=qo.suspense;qo.suspense=void 0===n?null:n;try{e(!1),t()}finally{qo.suspense=i}}))}function ya(e,n,t){var i=es(),r=vo.suspense;r={expirationTime:i=ns(i,e,r),suspenseConfig:r,action:t,eagerReducer:null,eagerState:null,next:null};var o=n.pending;if(null===o?r.next=r:(r.next=o.next,o.next=r),n.pending=r,o=e.alternate,e===Wo||null!==o&&o===Wo)Yo=!0,r.expirationTime=Ho,Wo.expirationTime=Ho;else{if(0===e.expirationTime&&(null===o||0===o.expirationTime)&&null!==(o=n.lastRenderedReducer))try{var a=n.lastRenderedState,l=o(a,t);if(r.eagerReducer=o,r.eagerState=l,Ui(l,a))return}catch(s){}ts(e,i)}}var ka={readContext:lo,useCallback:Qo,useContext:Qo,useEffect:Qo,useImperativeHandle:Qo,useLayoutEffect:Qo,useMemo:Qo,useReducer:Qo,useRef:Qo,useState:Qo,useDebugValue:Qo,useResponder:Qo,useDeferredValue:Qo,useTransition:Qo},wa={readContext:lo,useCallback:ga,useContext:lo,useEffect:ca,useImperativeHandle:function(e,n,t){return t=null!=t?t.concat([e]):null,la(4,2,da.bind(null,n,e),t)},useLayoutEffect:function(e,n){return la(4,2,e,n)},useMemo:function(e,n){var t=Zo();return n=void 0===n?null:n,e=e(),t.memoizedState=[e,n],e},useReducer:function(e,n,t){var i=Zo();return n=void 0!==t?t(n):n,i.memoizedState=i.baseState=n,e=(e=i.queue={pending:null,dispatch:null,lastRenderedReducer:e,lastRenderedState:n}).dispatch=ya.bind(null,Wo,e),[i.memoizedState,e]},useRef:function(e){return e={current:e},Zo().memoizedState=e},useState:ra,useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ra(e),i=t[0],r=t[1];return ca((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ra(!1),t=n[0];return n=n[1],[ga(ba.bind(null,n,e),[n,e]),t]}},_a={readContext:lo,useCallback:ha,useContext:lo,useEffect:ua,useImperativeHandle:fa,useLayoutEffect:pa,useMemo:va,useReducer:ta,useRef:aa,useState:function(){return ta(na)},useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ta(na),i=t[0],r=t[1];return ua((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ta(na),t=n[0];return n=n[1],[ha(ba.bind(null,n,e),[n,e]),t]}},xa={readContext:lo,useCallback:ha,useContext:lo,useEffect:ua,useImperativeHandle:fa,useLayoutEffect:pa,useMemo:va,useReducer:ia,useRef:aa,useState:function(){return ia(na)},useDebugValue:ma,useResponder:$o,useDeferredValue:function(e,n){var t=ia(na),i=t[0],r=t[1];return ua((function(){var t=qo.suspense;qo.suspense=void 0===n?null:n;try{r(e)}finally{qo.suspense=t}}),[e,n]),i},useTransition:function(e){var n=ia(na),t=n[0];return n=n[1],[ha(ba.bind(null,n,e),[n,e]),t]}},Ea=null,Sa=null,Ta=!1;function Pa(e,n){var t=Ns(5,null,null,0);t.elementType="DELETED",t.type="DELETED",t.stateNode=n,t.return=e,t.effectTag=8,null!==e.lastEffect?(e.lastEffect.nextEffect=t,e.lastEffect=t):e.firstEffect=e.lastEffect=t}function Ca(e,n){switch(e.tag){case 5:var t=e.type;return null!==(n=1!==n.nodeType||t.toLowerCase()!==n.nodeName.toLowerCase()?null:n)&&(e.stateNode=n,!0);case 6:return null!==(n=""===e.pendingProps||3!==n.nodeType?null:n)&&(e.stateNode=n,!0);case 13:default:return!1}}function Oa(e){if(Ta){var n=Sa;if(n){var t=n;if(!Ca(e,n)){if(!(n=St(t.nextSibling))||!Ca(e,n))return e.effectTag=-1025&e.effectTag|2,Ta=!1,void(Ea=e);Pa(Ea,t)}Ea=e,Sa=St(n.firstChild)}else e.effectTag=-1025&e.effectTag|2,Ta=!1,Ea=e}}function Aa(e){for(e=e.return;null!==e&&5!==e.tag&&3!==e.tag&&13!==e.tag;)e=e.return;Ea=e}function Ra(e){if(e!==Ea)return!1;if(!Ta)return Aa(e),Ta=!0,!1;var n=e.type;if(5!==e.tag||"head"!==n&&"body"!==n&&!_t(n,e.memoizedProps))for(n=Sa;n;)Pa(e,n),n=St(n.nextSibling);if(Aa(e),13===e.tag){if(!(e=null!==(e=e.memoizedState)?e.dehydrated:null))throw Error(a(317));e:{for(e=e.nextSibling,n=0;e;){if(8===e.nodeType){var t=e.data;if("/$"===t){if(0===n){Sa=St(e.nextSibling);break e}n--}else"$"!==t&&t!==bt&&t!==vt||n++}e=e.nextSibling}Sa=null}}else Sa=Ea?St(e.stateNode.nextSibling):null;return!0}function Na(){Sa=Ea=null,Ta=!1}var Ia=Q.ReactCurrentOwner,La=!1;function Da(e,n,t,i){n.child=null===e?Ao(n,null,t,i):Oo(n,e.child,t,i)}function ja(e,n,t,i,r){t=t.render;var o=n.ref;return ao(n,r),i=Jo(e,n,t,i,o,r),null===e||La?(n.effectTag|=1,Da(e,n,i,r),n.child):(n.updateQueue=e.updateQueue,n.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),Ja(e,n,r))}function Ma(e,n,t,i,r,o){if(null===e){var a=t.type;return"function"!=typeof a||Is(a)||void 0!==a.defaultProps||null!==t.compare||void 0!==t.defaultProps?((e=Ds(t.type,null,i,null,n.mode,o)).ref=n.ref,e.return=n,n.child=e):(n.tag=15,n.type=a,Fa(e,n,a,i,r,o))}return a=e.child,r<o&&(r=a.memoizedProps,(t=null!==(t=t.compare)?t:Gi)(r,i)&&e.ref===n.ref)?Ja(e,n,o):(n.effectTag|=1,(e=Ls(a,i)).ref=n.ref,e.return=n,n.child=e)}function Fa(e,n,t,i,r,o){return null!==e&&Gi(e.memoizedProps,i)&&e.ref===n.ref&&(La=!1,r<o)?(n.expirationTime=e.expirationTime,Ja(e,n,o)):za(e,n,t,i,o)}function Ba(e,n){var t=n.ref;(null===e&&null!==t||null!==e&&e.ref!==t)&&(n.effectTag|=128)}function za(e,n,t,i,r){var o=yr(t)?vr:gr.current;return o=br(n,o),ao(n,r),t=Jo(e,n,t,i,o,r),null===e||La?(n.effectTag|=1,Da(e,n,t,r),n.child):(n.updateQueue=e.updateQueue,n.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),Ja(e,n,r))}function Ua(e,n,t,i,r){if(yr(t)){var o=!0;xr(n)}else o=!1;if(ao(n,r),null===n.stateNode)null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),_o(n,t,i),Eo(n,t,i,r),i=!0;else if(null===e){var a=n.stateNode,l=n.memoizedProps;a.props=l;var s=a.context,c=t.contextType;"object"==typeof c&&null!==c?c=lo(c):c=br(n,c=yr(t)?vr:gr.current);var u=t.getDerivedStateFromProps,p="function"==typeof u||"function"==typeof a.getSnapshotBeforeUpdate;p||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(l!==i||s!==c)&&xo(n,a,i,c),so=!1;var d=n.memoizedState;a.state=d,go(n,i,a,r),s=n.memoizedState,l!==i||d!==s||hr.current||so?("function"==typeof u&&(yo(n,t,u,i),s=n.memoizedState),(l=so||wo(n,t,l,i,d,s,c))?(p||"function"!=typeof a.UNSAFE_componentWillMount&&"function"!=typeof a.componentWillMount||("function"==typeof a.componentWillMount&&a.componentWillMount(),"function"==typeof a.UNSAFE_componentWillMount&&a.UNSAFE_componentWillMount()),"function"==typeof a.componentDidMount&&(n.effectTag|=4)):("function"==typeof a.componentDidMount&&(n.effectTag|=4),n.memoizedProps=i,n.memoizedState=s),a.props=i,a.state=s,a.context=c,i=l):("function"==typeof a.componentDidMount&&(n.effectTag|=4),i=!1)}else a=n.stateNode,uo(e,n),l=n.memoizedProps,a.props=n.type===n.elementType?l:Jr(n.type,l),s=a.context,"object"==typeof(c=t.contextType)&&null!==c?c=lo(c):c=br(n,c=yr(t)?vr:gr.current),(p="function"==typeof(u=t.getDerivedStateFromProps)||"function"==typeof a.getSnapshotBeforeUpdate)||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(l!==i||s!==c)&&xo(n,a,i,c),so=!1,s=n.memoizedState,a.state=s,go(n,i,a,r),d=n.memoizedState,l!==i||s!==d||hr.current||so?("function"==typeof u&&(yo(n,t,u,i),d=n.memoizedState),(u=so||wo(n,t,l,i,s,d,c))?(p||"function"!=typeof a.UNSAFE_componentWillUpdate&&"function"!=typeof a.componentWillUpdate||("function"==typeof a.componentWillUpdate&&a.componentWillUpdate(i,d,c),"function"==typeof a.UNSAFE_componentWillUpdate&&a.UNSAFE_componentWillUpdate(i,d,c)),"function"==typeof a.componentDidUpdate&&(n.effectTag|=4),"function"==typeof a.getSnapshotBeforeUpdate&&(n.effectTag|=256)):("function"!=typeof a.componentDidUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=256),n.memoizedProps=i,n.memoizedState=d),a.props=i,a.state=d,a.context=c,i=u):("function"!=typeof a.componentDidUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||l===e.memoizedProps&&s===e.memoizedState||(n.effectTag|=256),i=!1);return $a(e,n,t,i,o,r)}function $a(e,n,t,i,r,o){Ba(e,n);var a=0!=(64&n.effectTag);if(!i&&!a)return r&&Er(n,t,!1),Ja(e,n,o);i=n.stateNode,Ia.current=n;var l=a&&"function"!=typeof t.getDerivedStateFromError?null:i.render();return n.effectTag|=1,null!==e&&a?(n.child=Oo(n,e.child,null,o),n.child=Oo(n,null,l,o)):Da(e,n,l,o),n.memoizedState=i.state,r&&Er(n,t,!0),n.child}function Ga(e){var n=e.stateNode;n.pendingContext?wr(0,n.pendingContext,n.pendingContext!==n.context):n.context&&wr(0,n.context,!1),jo(e,n.containerInfo)}var qa,Ha,Wa,Va={dehydrated:null,retryTime:0};function Ka(e,n,t){var i,r=n.mode,o=n.pendingProps,a=zo.current,l=!1;if((i=0!=(64&n.effectTag))||(i=0!=(2&a)&&(null===e||null!==e.memoizedState)),i?(l=!0,n.effectTag&=-65):null!==e&&null===e.memoizedState||void 0===o.fallback||!0===o.unstable_avoidThisFallback||(a|=1),fr(zo,1&a),null===e){if(void 0!==o.fallback&&Oa(n),l){if(l=o.fallback,(o=js(null,r,0,null)).return=n,0==(2&n.mode))for(e=null!==n.memoizedState?n.child.child:n.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(t=js(l,r,t,null)).return=n,o.sibling=t,n.memoizedState=Va,n.child=o,t}return r=o.children,n.memoizedState=null,n.child=Ao(n,null,r,t)}if(null!==e.memoizedState){if(r=(e=e.child).sibling,l){if(o=o.fallback,(t=Ls(e,e.pendingProps)).return=n,0==(2&n.mode)&&(l=null!==n.memoizedState?n.child.child:n.child)!==e.child)for(t.child=l;null!==l;)l.return=t,l=l.sibling;return(r=Ls(r,o)).return=n,t.sibling=r,t.childExpirationTime=0,n.memoizedState=Va,n.child=t,r}return t=Oo(n,e.child,o.children,t),n.memoizedState=null,n.child=t}if(e=e.child,l){if(l=o.fallback,(o=js(null,r,0,null)).return=n,o.child=e,null!==e&&(e.return=o),0==(2&n.mode))for(e=null!==n.memoizedState?n.child.child:n.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(t=js(l,r,t,null)).return=n,o.sibling=t,t.effectTag|=2,o.childExpirationTime=0,n.memoizedState=Va,n.child=o,t}return n.memoizedState=null,n.child=Oo(n,e,o.children,t)}function Ya(e,n){e.expirationTime<n&&(e.expirationTime=n);var t=e.alternate;null!==t&&t.expirationTime<n&&(t.expirationTime=n),oo(e.return,n)}function Qa(e,n,t,i,r,o){var a=e.memoizedState;null===a?e.memoizedState={isBackwards:n,rendering:null,renderingStartTime:0,last:i,tail:t,tailExpiration:0,tailMode:r,lastEffect:o}:(a.isBackwards=n,a.rendering=null,a.renderingStartTime=0,a.last=i,a.tail=t,a.tailExpiration=0,a.tailMode=r,a.lastEffect=o)}function Xa(e,n,t){var i=n.pendingProps,r=i.revealOrder,o=i.tail;if(Da(e,n,i.children,t),0!=(2&(i=zo.current)))i=1&i|2,n.effectTag|=64;else{if(null!==e&&0!=(64&e.effectTag))e:for(e=n.child;null!==e;){if(13===e.tag)null!==e.memoizedState&&Ya(e,t);else if(19===e.tag)Ya(e,t);else if(null!==e.child){e.child.return=e,e=e.child;continue}if(e===n)break e;for(;null===e.sibling;){if(null===e.return||e.return===n)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}i&=1}if(fr(zo,i),0==(2&n.mode))n.memoizedState=null;else switch(r){case"forwards":for(t=n.child,r=null;null!==t;)null!==(e=t.alternate)&&null===Uo(e)&&(r=t),t=t.sibling;null===(t=r)?(r=n.child,n.child=null):(r=t.sibling,t.sibling=null),Qa(n,!1,r,t,o,n.lastEffect);break;case"backwards":for(t=null,r=n.child,n.child=null;null!==r;){if(null!==(e=r.alternate)&&null===Uo(e)){n.child=r;break}e=r.sibling,r.sibling=t,t=r,r=e}Qa(n,!0,t,null,o,n.lastEffect);break;case"together":Qa(n,!1,null,null,void 0,n.lastEffect);break;default:n.memoizedState=null}return n.child}function Ja(e,n,t){null!==e&&(n.dependencies=e.dependencies);var i=n.expirationTime;if(0!==i&&ms(i),n.childExpirationTime<t)return null;if(null!==e&&n.child!==e.child)throw Error(a(153));if(null!==n.child){for(t=Ls(e=n.child,e.pendingProps),n.child=t,t.return=n;null!==e.sibling;)e=e.sibling,(t=t.sibling=Ls(e,e.pendingProps)).return=n;t.sibling=null}return n.child}function Za(e,n){switch(e.tailMode){case"hidden":n=e.tail;for(var t=null;null!==n;)null!==n.alternate&&(t=n),n=n.sibling;null===t?e.tail=null:t.sibling=null;break;case"collapsed":t=e.tail;for(var i=null;null!==t;)null!==t.alternate&&(i=t),t=t.sibling;null===i?n||null===e.tail?e.tail=null:e.tail.sibling=null:i.sibling=null}}function el(e,n,t){var i=n.pendingProps;switch(n.tag){case 2:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return null;case 1:return yr(n.type)&&kr(),null;case 3:return Mo(),dr(hr),dr(gr),(t=n.stateNode).pendingContext&&(t.context=t.pendingContext,t.pendingContext=null),null!==e&&null!==e.child||!Ra(n)||(n.effectTag|=4),null;case 5:Bo(n),t=Do(Lo.current);var o=n.type;if(null!==e&&null!=n.stateNode)Ha(e,n,o,i,t),e.ref!==n.ref&&(n.effectTag|=128);else{if(!i){if(null===n.stateNode)throw Error(a(166));return null}if(e=Do(No.current),Ra(n)){i=n.stateNode,o=n.type;var l=n.memoizedProps;switch(i[Ct]=n,i[Ot]=l,o){case"iframe":case"object":case"embed":Yn("load",i);break;case"video":case"audio":for(e=0;e<Xe.length;e++)Yn(Xe[e],i);break;case"source":Yn("error",i);break;case"img":case"image":case"link":Yn("error",i),Yn("load",i);break;case"form":Yn("reset",i),Yn("submit",i);break;case"details":Yn("toggle",i);break;case"input":xe(i,l),Yn("invalid",i),ct(t,"onChange");break;case"select":i._wrapperState={wasMultiple:!!l.multiple},Yn("invalid",i),ct(t,"onChange");break;case"textarea":Re(i,l),Yn("invalid",i),ct(t,"onChange")}for(var s in at(o,l),e=null,l)if(l.hasOwnProperty(s)){var c=l[s];"children"===s?"string"==typeof c?i.textContent!==c&&(e=["children",c]):"number"==typeof c&&i.textContent!==""+c&&(e=["children",""+c]):E.hasOwnProperty(s)&&null!=c&&ct(t,s)}switch(o){case"input":ke(i),Te(i,l,!0);break;case"textarea":ke(i),Ie(i);break;case"select":case"option":break;default:"function"==typeof l.onClick&&(i.onclick=ut)}t=e,n.updateQueue=t,null!==t&&(n.effectTag|=4)}else{switch(s=9===t.nodeType?t:t.ownerDocument,e===st&&(e=je(o)),e===st?"script"===o?((e=s.createElement("div")).innerHTML="<script><\/script>",e=e.removeChild(e.firstChild)):"string"==typeof i.is?e=s.createElement(o,{is:i.is}):(e=s.createElement(o),"select"===o&&(s=e,i.multiple?s.multiple=!0:i.size&&(s.size=i.size))):e=s.createElementNS(e,o),e[Ct]=n,e[Ot]=i,qa(e,n),n.stateNode=e,s=lt(o,i),o){case"iframe":case"object":case"embed":Yn("load",e),c=i;break;case"video":case"audio":for(c=0;c<Xe.length;c++)Yn(Xe[c],e);c=i;break;case"source":Yn("error",e),c=i;break;case"img":case"image":case"link":Yn("error",e),Yn("load",e),c=i;break;case"form":Yn("reset",e),Yn("submit",e),c=i;break;case"details":Yn("toggle",e),c=i;break;case"input":xe(e,i),c=_e(e,i),Yn("invalid",e),ct(t,"onChange");break;case"option":c=Ce(e,i);break;case"select":e._wrapperState={wasMultiple:!!i.multiple},c=r({},i,{value:void 0}),Yn("invalid",e),ct(t,"onChange");break;case"textarea":Re(e,i),c=Ae(e,i),Yn("invalid",e),ct(t,"onChange");break;default:c=i}at(o,c);var u=c;for(l in u)if(u.hasOwnProperty(l)){var p=u[l];"style"===l?rt(e,p):"dangerouslySetInnerHTML"===l?null!=(p=p?p.__html:void 0)&&ze(e,p):"children"===l?"string"==typeof p?("textarea"!==o||""!==p)&&Ue(e,p):"number"==typeof p&&Ue(e,""+p):"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&"autoFocus"!==l&&(E.hasOwnProperty(l)?null!=p&&ct(t,l):null!=p&&X(e,l,p,s))}switch(o){case"input":ke(e),Te(e,i,!1);break;case"textarea":ke(e),Ie(e);break;case"option":null!=i.value&&e.setAttribute("value",""+be(i.value));break;case"select":e.multiple=!!i.multiple,null!=(t=i.value)?Oe(e,!!i.multiple,t,!1):null!=i.defaultValue&&Oe(e,!!i.multiple,i.defaultValue,!0);break;default:"function"==typeof c.onClick&&(e.onclick=ut)}wt(o,i)&&(n.effectTag|=4)}null!==n.ref&&(n.effectTag|=128)}return null;case 6:if(e&&null!=n.stateNode)Wa(0,n,e.memoizedProps,i);else{if("string"!=typeof i&&null===n.stateNode)throw Error(a(166));t=Do(Lo.current),Do(No.current),Ra(n)?(t=n.stateNode,i=n.memoizedProps,t[Ct]=n,t.nodeValue!==i&&(n.effectTag|=4)):((t=(9===t.nodeType?t:t.ownerDocument).createTextNode(i))[Ct]=n,n.stateNode=t)}return null;case 13:return dr(zo),i=n.memoizedState,0!=(64&n.effectTag)?(n.expirationTime=t,n):(t=null!==i,i=!1,null===e?void 0!==n.memoizedProps.fallback&&Ra(n):(i=null!==(o=e.memoizedState),t||null===o||null!==(o=e.child.sibling)&&(null!==(l=n.firstEffect)?(n.firstEffect=o,o.nextEffect=l):(n.firstEffect=n.lastEffect=o,o.nextEffect=null),o.effectTag=8)),t&&!i&&0!=(2&n.mode)&&(null===e&&!0!==n.memoizedProps.unstable_avoidThisFallback||0!=(1&zo.current)?Dl===Cl&&(Dl=Ol):(Dl!==Cl&&Dl!==Ol||(Dl=Al),0!==zl&&null!==Nl&&(Us(Nl,Ll),$s(Nl,zl)))),(t||i)&&(n.effectTag|=4),null);case 4:return Mo(),null;case 10:return ro(n),null;case 17:return yr(n.type)&&kr(),null;case 19:if(dr(zo),null===(i=n.memoizedState))return null;if(o=0!=(64&n.effectTag),null===(l=i.rendering)){if(o)Za(i,!1);else if(Dl!==Cl||null!==e&&0!=(64&e.effectTag))for(l=n.child;null!==l;){if(null!==(e=Uo(l))){for(n.effectTag|=64,Za(i,!1),null!==(o=e.updateQueue)&&(n.updateQueue=o,n.effectTag|=4),null===i.lastEffect&&(n.firstEffect=null),n.lastEffect=i.lastEffect,i=n.child;null!==i;)l=t,(o=i).effectTag&=2,o.nextEffect=null,o.firstEffect=null,o.lastEffect=null,null===(e=o.alternate)?(o.childExpirationTime=0,o.expirationTime=l,o.child=null,o.memoizedProps=null,o.memoizedState=null,o.updateQueue=null,o.dependencies=null):(o.childExpirationTime=e.childExpirationTime,o.expirationTime=e.expirationTime,o.child=e.child,o.memoizedProps=e.memoizedProps,o.memoizedState=e.memoizedState,o.updateQueue=e.updateQueue,l=e.dependencies,o.dependencies=null===l?null:{expirationTime:l.expirationTime,firstContext:l.firstContext,responders:l.responders}),i=i.sibling;return fr(zo,1&zo.current|2),n.child}l=l.sibling}}else{if(!o)if(null!==(e=Uo(l))){if(n.effectTag|=64,o=!0,null!==(t=e.updateQueue)&&(n.updateQueue=t,n.effectTag|=4),Za(i,!0),null===i.tail&&"hidden"===i.tailMode&&!l.alternate)return null!==(n=n.lastEffect=i.lastEffect)&&(n.nextEffect=null),null}else 2*Gr()-i.renderingStartTime>i.tailExpiration&&1<t&&(n.effectTag|=64,o=!0,Za(i,!1),n.expirationTime=n.childExpirationTime=t-1);i.isBackwards?(l.sibling=n.child,n.child=l):(null!==(t=i.last)?t.sibling=l:n.child=l,i.last=l)}return null!==i.tail?(0===i.tailExpiration&&(i.tailExpiration=Gr()+500),t=i.tail,i.rendering=t,i.tail=t.sibling,i.lastEffect=n.lastEffect,i.renderingStartTime=Gr(),t.sibling=null,n=zo.current,fr(zo,o?1&n|2:1&n),t):null}throw Error(a(156,n.tag))}function nl(e){switch(e.tag){case 1:yr(e.type)&&kr();var n=e.effectTag;return 4096&n?(e.effectTag=-4097&n|64,e):null;case 3:if(Mo(),dr(hr),dr(gr),0!=(64&(n=e.effectTag)))throw Error(a(285));return e.effectTag=-4097&n|64,e;case 5:return Bo(e),null;case 13:return dr(zo),4096&(n=e.effectTag)?(e.effectTag=-4097&n|64,e):null;case 19:return dr(zo),null;case 4:return Mo(),null;case 10:return ro(e),null;default:return null}}function tl(e,n){return{value:e,source:n,stack:ve(n)}}qa=function(e,n){for(var t=n.child;null!==t;){if(5===t.tag||6===t.tag)e.appendChild(t.stateNode);else if(4!==t.tag&&null!==t.child){t.child.return=t,t=t.child;continue}if(t===n)break;for(;null===t.sibling;){if(null===t.return||t.return===n)return;t=t.return}t.sibling.return=t.return,t=t.sibling}},Ha=function(e,n,t,i,o){var a=e.memoizedProps;if(a!==i){var l,s,c=n.stateNode;switch(Do(No.current),e=null,t){case"input":a=_e(c,a),i=_e(c,i),e=[];break;case"option":a=Ce(c,a),i=Ce(c,i),e=[];break;case"select":a=r({},a,{value:void 0}),i=r({},i,{value:void 0}),e=[];break;case"textarea":a=Ae(c,a),i=Ae(c,i),e=[];break;default:"function"!=typeof a.onClick&&"function"==typeof i.onClick&&(c.onclick=ut)}for(l in at(t,i),t=null,a)if(!i.hasOwnProperty(l)&&a.hasOwnProperty(l)&&null!=a[l])if("style"===l)for(s in c=a[l])c.hasOwnProperty(s)&&(t||(t={}),t[s]="");else"dangerouslySetInnerHTML"!==l&&"children"!==l&&"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&"autoFocus"!==l&&(E.hasOwnProperty(l)?e||(e=[]):(e=e||[]).push(l,null));for(l in i){var u=i[l];if(c=null!=a?a[l]:void 0,i.hasOwnProperty(l)&&u!==c&&(null!=u||null!=c))if("style"===l)if(c){for(s in c)!c.hasOwnProperty(s)||u&&u.hasOwnProperty(s)||(t||(t={}),t[s]="");for(s in u)u.hasOwnProperty(s)&&c[s]!==u[s]&&(t||(t={}),t[s]=u[s])}else t||(e||(e=[]),e.push(l,t)),t=u;else"dangerouslySetInnerHTML"===l?(u=u?u.__html:void 0,c=c?c.__html:void 0,null!=u&&c!==u&&(e=e||[]).push(l,u)):"children"===l?c===u||"string"!=typeof u&&"number"!=typeof u||(e=e||[]).push(l,""+u):"suppressContentEditableWarning"!==l&&"suppressHydrationWarning"!==l&&(E.hasOwnProperty(l)?(null!=u&&ct(o,l),e||c===u||(e=[])):(e=e||[]).push(l,u))}t&&(e=e||[]).push("style",t),o=e,(n.updateQueue=o)&&(n.effectTag|=4)}},Wa=function(e,n,t,i){t!==i&&(n.effectTag|=4)};var il="function"==typeof WeakSet?WeakSet:Set;function rl(e,n){var t=n.source,i=n.stack;null===i&&null!==t&&(i=ve(t)),null!==t&&he(t.type),n=n.value,null!==e&&1===e.tag&&he(e.type);try{console.error(n)}catch(r){setTimeout((function(){throw r}))}}function ol(e){var n=e.ref;if(null!==n)if("function"==typeof n)try{n(null)}catch(t){Ts(e,t)}else n.current=null}function al(e,n){switch(n.tag){case 0:case 11:case 15:case 22:return;case 1:if(256&n.effectTag&&null!==e){var t=e.memoizedProps,i=e.memoizedState;n=(e=n.stateNode).getSnapshotBeforeUpdate(n.elementType===n.type?t:Jr(n.type,t),i),e.__reactInternalSnapshotBeforeUpdate=n}return;case 3:case 5:case 6:case 4:case 17:return}throw Error(a(163))}function ll(e,n){if(null!==(n=null!==(n=n.updateQueue)?n.lastEffect:null)){var t=n=n.next;do{if((t.tag&e)===e){var i=t.destroy;t.destroy=void 0,void 0!==i&&i()}t=t.next}while(t!==n)}}function sl(e,n){if(null!==(n=null!==(n=n.updateQueue)?n.lastEffect:null)){var t=n=n.next;do{if((t.tag&e)===e){var i=t.create;t.destroy=i()}t=t.next}while(t!==n)}}function cl(e,n,t){switch(t.tag){case 0:case 11:case 15:case 22:return void sl(3,t);case 1:if(e=t.stateNode,4&t.effectTag)if(null===n)e.componentDidMount();else{var i=t.elementType===t.type?n.memoizedProps:Jr(t.type,n.memoizedProps);e.componentDidUpdate(i,n.memoizedState,e.__reactInternalSnapshotBeforeUpdate)}return void(null!==(n=t.updateQueue)&&ho(t,n,e));case 3:if(null!==(n=t.updateQueue)){if(e=null,null!==t.child)switch(t.child.tag){case 5:e=t.child.stateNode;break;case 1:e=t.child.stateNode}ho(t,n,e)}return;case 5:return e=t.stateNode,void(null===n&&4&t.effectTag&&wt(t.type,t.memoizedProps)&&e.focus());case 6:case 4:case 12:return;case 13:return void(null===t.memoizedState&&(t=t.alternate,null!==t&&(t=t.memoizedState,null!==t&&(t=t.dehydrated,null!==t&&Fn(t)))));case 19:case 17:case 20:case 21:return}throw Error(a(163))}function ul(e,n,t){switch("function"==typeof As&&As(n),n.tag){case 0:case 11:case 14:case 15:case 22:if(null!==(e=n.updateQueue)&&null!==(e=e.lastEffect)){var i=e.next;Wr(97<t?97:t,(function(){var e=i;do{var t=e.destroy;if(void 0!==t){var r=n;try{t()}catch(o){Ts(r,o)}}e=e.next}while(e!==i)}))}break;case 1:ol(n),"function"==typeof(t=n.stateNode).componentWillUnmount&&function(e,n){try{n.props=e.memoizedProps,n.state=e.memoizedState,n.componentWillUnmount()}catch(t){Ts(e,t)}}(n,t);break;case 5:ol(n);break;case 4:hl(e,n,t)}}function pl(e){var n=e.alternate;e.return=null,e.child=null,e.memoizedState=null,e.updateQueue=null,e.dependencies=null,e.alternate=null,e.firstEffect=null,e.lastEffect=null,e.pendingProps=null,e.memoizedProps=null,e.stateNode=null,null!==n&&pl(n)}function dl(e){return 5===e.tag||3===e.tag||4===e.tag}function fl(e){e:{for(var n=e.return;null!==n;){if(dl(n)){var t=n;break e}n=n.return}throw Error(a(160))}switch(n=t.stateNode,t.tag){case 5:var i=!1;break;case 3:case 4:n=n.containerInfo,i=!0;break;default:throw Error(a(161))}16&t.effectTag&&(Ue(n,""),t.effectTag&=-17);e:n:for(t=e;;){for(;null===t.sibling;){if(null===t.return||dl(t.return)){t=null;break e}t=t.return}for(t.sibling.return=t.return,t=t.sibling;5!==t.tag&&6!==t.tag&&18!==t.tag;){if(2&t.effectTag)continue n;if(null===t.child||4===t.tag)continue n;t.child.return=t,t=t.child}if(!(2&t.effectTag)){t=t.stateNode;break e}}i?ml(e,t,n):gl(e,t,n)}function ml(e,n,t){var i=e.tag,r=5===i||6===i;if(r)e=r?e.stateNode:e.stateNode.instance,n?8===t.nodeType?t.parentNode.insertBefore(e,n):t.insertBefore(e,n):(8===t.nodeType?(n=t.parentNode).insertBefore(e,t):(n=t).appendChild(e),null!=(t=t._reactRootContainer)||null!==n.onclick||(n.onclick=ut));else if(4!==i&&null!==(e=e.child))for(ml(e,n,t),e=e.sibling;null!==e;)ml(e,n,t),e=e.sibling}function gl(e,n,t){var i=e.tag,r=5===i||6===i;if(r)e=r?e.stateNode:e.stateNode.instance,n?t.insertBefore(e,n):t.appendChild(e);else if(4!==i&&null!==(e=e.child))for(gl(e,n,t),e=e.sibling;null!==e;)gl(e,n,t),e=e.sibling}function hl(e,n,t){for(var i,r,o=n,l=!1;;){if(!l){l=o.return;e:for(;;){if(null===l)throw Error(a(160));switch(i=l.stateNode,l.tag){case 5:r=!1;break e;case 3:case 4:i=i.containerInfo,r=!0;break e}l=l.return}l=!0}if(5===o.tag||6===o.tag){e:for(var s=e,c=o,u=t,p=c;;)if(ul(s,p,u),null!==p.child&&4!==p.tag)p.child.return=p,p=p.child;else{if(p===c)break e;for(;null===p.sibling;){if(null===p.return||p.return===c)break e;p=p.return}p.sibling.return=p.return,p=p.sibling}r?(s=i,c=o.stateNode,8===s.nodeType?s.parentNode.removeChild(c):s.removeChild(c)):i.removeChild(o.stateNode)}else if(4===o.tag){if(null!==o.child){i=o.stateNode.containerInfo,r=!0,o.child.return=o,o=o.child;continue}}else if(ul(e,o,t),null!==o.child){o.child.return=o,o=o.child;continue}if(o===n)break;for(;null===o.sibling;){if(null===o.return||o.return===n)return;4===(o=o.return).tag&&(l=!1)}o.sibling.return=o.return,o=o.sibling}}function vl(e,n){switch(n.tag){case 0:case 11:case 14:case 15:case 22:return void ll(3,n);case 1:return;case 5:var t=n.stateNode;if(null!=t){var i=n.memoizedProps,r=null!==e?e.memoizedProps:i;e=n.type;var o=n.updateQueue;if(n.updateQueue=null,null!==o){for(t[Ot]=i,"input"===e&&"radio"===i.type&&null!=i.name&&Ee(t,i),lt(e,r),n=lt(e,i),r=0;r<o.length;r+=2){var l=o[r],s=o[r+1];"style"===l?rt(t,s):"dangerouslySetInnerHTML"===l?ze(t,s):"children"===l?Ue(t,s):X(t,l,s,n)}switch(e){case"input":Se(t,i);break;case"textarea":Ne(t,i);break;case"select":n=t._wrapperState.wasMultiple,t._wrapperState.wasMultiple=!!i.multiple,null!=(e=i.value)?Oe(t,!!i.multiple,e,!1):n!==!!i.multiple&&(null!=i.defaultValue?Oe(t,!!i.multiple,i.defaultValue,!0):Oe(t,!!i.multiple,i.multiple?[]:"",!1))}}}return;case 6:if(null===n.stateNode)throw Error(a(162));return void(n.stateNode.nodeValue=n.memoizedProps);case 3:return void((n=n.stateNode).hydrate&&(n.hydrate=!1,Fn(n.containerInfo)));case 12:return;case 13:if(t=n,null===n.memoizedState?i=!1:(i=!0,t=n.child,$l=Gr()),null!==t)e:for(e=t;;){if(5===e.tag)o=e.stateNode,i?"function"==typeof(o=o.style).setProperty?o.setProperty("display","none","important"):o.display="none":(o=e.stateNode,r=null!=(r=e.memoizedProps.style)&&r.hasOwnProperty("display")?r.display:null,o.style.display=it("display",r));else if(6===e.tag)e.stateNode.nodeValue=i?"":e.memoizedProps;else{if(13===e.tag&&null!==e.memoizedState&&null===e.memoizedState.dehydrated){(o=e.child.sibling).return=e,e=o;continue}if(null!==e.child){e.child.return=e,e=e.child;continue}}if(e===t)break;for(;null===e.sibling;){if(null===e.return||e.return===t)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}return void bl(n);case 19:return void bl(n);case 17:return}throw Error(a(163))}function bl(e){var n=e.updateQueue;if(null!==n){e.updateQueue=null;var t=e.stateNode;null===t&&(t=e.stateNode=new il),n.forEach((function(n){var i=Cs.bind(null,e,n);t.has(n)||(t.add(n),n.then(i,i))}))}}var yl="function"==typeof WeakMap?WeakMap:Map;function kl(e,n,t){(t=po(t,null)).tag=3,t.payload={element:null};var i=n.value;return t.callback=function(){ql||(ql=!0,Hl=i),rl(e,n)},t}function wl(e,n,t){(t=po(t,null)).tag=3;var i=e.type.getDerivedStateFromError;if("function"==typeof i){var r=n.value;t.payload=function(){return rl(e,n),i(r)}}var o=e.stateNode;return null!==o&&"function"==typeof o.componentDidCatch&&(t.callback=function(){"function"!=typeof i&&(null===Wl?Wl=new Set([this]):Wl.add(this),rl(e,n));var t=n.stack;this.componentDidCatch(n.value,{componentStack:null!==t?t:""})}),t}var _l,xl=Math.ceil,El=Q.ReactCurrentDispatcher,Sl=Q.ReactCurrentOwner,Tl=16,Pl=32,Cl=0,Ol=3,Al=4,Rl=0,Nl=null,Il=null,Ll=0,Dl=Cl,jl=null,Ml=1073741823,Fl=1073741823,Bl=null,zl=0,Ul=!1,$l=0,Gl=null,ql=!1,Hl=null,Wl=null,Vl=!1,Kl=null,Yl=90,Ql=null,Xl=0,Jl=null,Zl=0;function es(){return 0!=(48&Rl)?1073741821-(Gr()/10|0):0!==Zl?Zl:Zl=1073741821-(Gr()/10|0)}function ns(e,n,t){if(0==(2&(n=n.mode)))return 1073741823;var i=qr();if(0==(4&n))return 99===i?1073741823:1073741822;if(0!=(Rl&Tl))return Ll;if(null!==t)e=Xr(e,0|t.timeoutMs||5e3,250);else switch(i){case 99:e=1073741823;break;case 98:e=Xr(e,150,100);break;case 97:case 96:e=Xr(e,5e3,250);break;case 95:e=2;break;default:throw Error(a(326))}return null!==Nl&&e===Ll&&--e,e}function ts(e,n){if(50<Xl)throw Xl=0,Jl=null,Error(a(185));if(null!==(e=is(e,n))){var t=qr();1073741823===n?0!=(8&Rl)&&0==(48&Rl)?ls(e):(os(e),0===Rl&&Yr()):os(e),0==(4&Rl)||98!==t&&99!==t||(null===Ql?Ql=new Map([[e,n]]):(void 0===(t=Ql.get(e))||t>n)&&Ql.set(e,n))}}function is(e,n){e.expirationTime<n&&(e.expirationTime=n);var t=e.alternate;null!==t&&t.expirationTime<n&&(t.expirationTime=n);var i=e.return,r=null;if(null===i&&3===e.tag)r=e.stateNode;else for(;null!==i;){if(t=i.alternate,i.childExpirationTime<n&&(i.childExpirationTime=n),null!==t&&t.childExpirationTime<n&&(t.childExpirationTime=n),null===i.return&&3===i.tag){r=i.stateNode;break}i=i.return}return null!==r&&(Nl===r&&(ms(n),Dl===Al&&Us(r,Ll)),$s(r,n)),r}function rs(e){var n=e.lastExpiredTime;if(0!==n)return n;if(!zs(e,n=e.firstPendingTime))return n;var t=e.lastPingedTime;return 2>=(e=t>(e=e.nextKnownPendingLevel)?t:e)&&n!==e?0:e}function os(e){if(0!==e.lastExpiredTime)e.callbackExpirationTime=1073741823,e.callbackPriority=99,e.callbackNode=Kr(ls.bind(null,e));else{var n=rs(e),t=e.callbackNode;if(0===n)null!==t&&(e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90);else{var i=es();if(1073741823===n?i=99:1===n||2===n?i=95:i=0>=(i=10*(1073741821-n)-10*(1073741821-i))?99:250>=i?98:5250>=i?97:95,null!==t){var r=e.callbackPriority;if(e.callbackExpirationTime===n&&r>=i)return;t!==jr&&Pr(t)}e.callbackExpirationTime=n,e.callbackPriority=i,n=1073741823===n?Kr(ls.bind(null,e)):Vr(i,as.bind(null,e),{timeout:10*(1073741821-n)-Gr()}),e.callbackNode=n}}}function as(e,n){if(Zl=0,n)return Gs(e,n=es()),os(e),null;var t=rs(e);if(0!==t){if(n=e.callbackNode,0!=(48&Rl))throw Error(a(327));if(xs(),e===Nl&&t===Ll||us(e,t),null!==Il){var i=Rl;Rl|=Tl;for(var r=ds();;)try{hs();break}catch(s){ps(e,s)}if(io(),Rl=i,El.current=r,1===Dl)throw n=jl,us(e,t),Us(e,t),os(e),n;if(null===Il)switch(r=e.finishedWork=e.current.alternate,e.finishedExpirationTime=t,i=Dl,Nl=null,i){case Cl:case 1:throw Error(a(345));case 2:Gs(e,2<t?2:t);break;case Ol:if(Us(e,t),t===(i=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=ys(r)),1073741823===Ml&&10<(r=$l+500-Gr())){if(Ul){var o=e.lastPingedTime;if(0===o||o>=t){e.lastPingedTime=t,us(e,t);break}}if(0!==(o=rs(e))&&o!==t)break;if(0!==i&&i!==t){e.lastPingedTime=i;break}e.timeoutHandle=xt(ks.bind(null,e),r);break}ks(e);break;case Al:if(Us(e,t),t===(i=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=ys(r)),Ul&&(0===(r=e.lastPingedTime)||r>=t)){e.lastPingedTime=t,us(e,t);break}if(0!==(r=rs(e))&&r!==t)break;if(0!==i&&i!==t){e.lastPingedTime=i;break}if(1073741823!==Fl?i=10*(1073741821-Fl)-Gr():1073741823===Ml?i=0:(i=10*(1073741821-Ml)-5e3,0>(i=(r=Gr())-i)&&(i=0),(t=10*(1073741821-t)-r)<(i=(120>i?120:480>i?480:1080>i?1080:1920>i?1920:3e3>i?3e3:4320>i?4320:1960*xl(i/1960))-i)&&(i=t)),10<i){e.timeoutHandle=xt(ks.bind(null,e),i);break}ks(e);break;case 5:if(1073741823!==Ml&&null!==Bl){o=Ml;var l=Bl;if(0>=(i=0|l.busyMinDurationMs)?i=0:(r=0|l.busyDelayMs,i=(o=Gr()-(10*(1073741821-o)-(0|l.timeoutMs||5e3)))<=r?0:r+i-o),10<i){Us(e,t),e.timeoutHandle=xt(ks.bind(null,e),i);break}}ks(e);break;default:throw Error(a(329))}if(os(e),e.callbackNode===n)return as.bind(null,e)}}return null}function ls(e){var n=e.lastExpiredTime;if(n=0!==n?n:1073741823,0!=(48&Rl))throw Error(a(327));if(xs(),e===Nl&&n===Ll||us(e,n),null!==Il){var t=Rl;Rl|=Tl;for(var i=ds();;)try{gs();break}catch(r){ps(e,r)}if(io(),Rl=t,El.current=i,1===Dl)throw t=jl,us(e,n),Us(e,n),os(e),t;if(null!==Il)throw Error(a(261));e.finishedWork=e.current.alternate,e.finishedExpirationTime=n,Nl=null,ks(e),os(e)}return null}function ss(e,n){var t=Rl;Rl|=1;try{return e(n)}finally{0===(Rl=t)&&Yr()}}function cs(e,n){var t=Rl;Rl&=-2,Rl|=8;try{return e(n)}finally{0===(Rl=t)&&Yr()}}function us(e,n){e.finishedWork=null,e.finishedExpirationTime=0;var t=e.timeoutHandle;if(-1!==t&&(e.timeoutHandle=-1,Et(t)),null!==Il)for(t=Il.return;null!==t;){var i=t;switch(i.tag){case 1:null!=(i=i.type.childContextTypes)&&kr();break;case 3:Mo(),dr(hr),dr(gr);break;case 5:Bo(i);break;case 4:Mo();break;case 13:case 19:dr(zo);break;case 10:ro(i)}t=t.return}Nl=e,Il=Ls(e.current,null),Ll=n,Dl=Cl,jl=null,Fl=Ml=1073741823,Bl=null,zl=0,Ul=!1}function ps(e,n){for(;;){try{if(io(),Go.current=ka,Yo)for(var t=Wo.memoizedState;null!==t;){var i=t.queue;null!==i&&(i.pending=null),t=t.next}if(Ho=0,Ko=Vo=Wo=null,Yo=!1,null===Il||null===Il.return)return Dl=1,jl=n,Il=null;e:{var r=e,o=Il.return,a=Il,l=n;if(n=Ll,a.effectTag|=2048,a.firstEffect=a.lastEffect=null,null!==l&&"object"==typeof l&&"function"==typeof l.then){var s=l;if(0==(2&a.mode)){var c=a.alternate;c?(a.updateQueue=c.updateQueue,a.memoizedState=c.memoizedState,a.expirationTime=c.expirationTime):(a.updateQueue=null,a.memoizedState=null)}var u=0!=(1&zo.current),p=o;do{var d;if(d=13===p.tag){var f=p.memoizedState;if(null!==f)d=null!==f.dehydrated;else{var m=p.memoizedProps;d=void 0!==m.fallback&&(!0!==m.unstable_avoidThisFallback||!u)}}if(d){var g=p.updateQueue;if(null===g){var h=new Set;h.add(s),p.updateQueue=h}else g.add(s);if(0==(2&p.mode)){if(p.effectTag|=64,a.effectTag&=-2981,1===a.tag)if(null===a.alternate)a.tag=17;else{var v=po(1073741823,null);v.tag=2,fo(a,v)}a.expirationTime=1073741823;break e}l=void 0,a=n;var b=r.pingCache;if(null===b?(b=r.pingCache=new yl,l=new Set,b.set(s,l)):void 0===(l=b.get(s))&&(l=new Set,b.set(s,l)),!l.has(a)){l.add(a);var y=Ps.bind(null,r,s,a);s.then(y,y)}p.effectTag|=4096,p.expirationTime=n;break e}p=p.return}while(null!==p);l=Error((he(a.type)||"A React component")+" suspended while rendering, but no fallback UI was specified.\n\nAdd a <Suspense fallback=...> component higher in the tree to provide a loading indicator or placeholder to display."+ve(a))}5!==Dl&&(Dl=2),l=tl(l,a),p=o;do{switch(p.tag){case 3:s=l,p.effectTag|=4096,p.expirationTime=n,mo(p,kl(p,s,n));break e;case 1:s=l;var k=p.type,w=p.stateNode;if(0==(64&p.effectTag)&&("function"==typeof k.getDerivedStateFromError||null!==w&&"function"==typeof w.componentDidCatch&&(null===Wl||!Wl.has(w)))){p.effectTag|=4096,p.expirationTime=n,mo(p,wl(p,s,n));break e}}p=p.return}while(null!==p)}Il=bs(Il)}catch(_){n=_;continue}break}}function ds(){var e=El.current;return El.current=ka,null===e?ka:e}function fs(e,n){e<Ml&&2<e&&(Ml=e),null!==n&&e<Fl&&2<e&&(Fl=e,Bl=n)}function ms(e){e>zl&&(zl=e)}function gs(){for(;null!==Il;)Il=vs(Il)}function hs(){for(;null!==Il&&!Mr();)Il=vs(Il)}function vs(e){var n=_l(e.alternate,e,Ll);return e.memoizedProps=e.pendingProps,null===n&&(n=bs(e)),Sl.current=null,n}function bs(e){Il=e;do{var n=Il.alternate;if(e=Il.return,0==(2048&Il.effectTag)){if(n=el(n,Il,Ll),1===Ll||1!==Il.childExpirationTime){for(var t=0,i=Il.child;null!==i;){var r=i.expirationTime,o=i.childExpirationTime;r>t&&(t=r),o>t&&(t=o),i=i.sibling}Il.childExpirationTime=t}if(null!==n)return n;null!==e&&0==(2048&e.effectTag)&&(null===e.firstEffect&&(e.firstEffect=Il.firstEffect),null!==Il.lastEffect&&(null!==e.lastEffect&&(e.lastEffect.nextEffect=Il.firstEffect),e.lastEffect=Il.lastEffect),1<Il.effectTag&&(null!==e.lastEffect?e.lastEffect.nextEffect=Il:e.firstEffect=Il,e.lastEffect=Il))}else{if(null!==(n=nl(Il)))return n.effectTag&=2047,n;null!==e&&(e.firstEffect=e.lastEffect=null,e.effectTag|=2048)}if(null!==(n=Il.sibling))return n;Il=e}while(null!==Il);return Dl===Cl&&(Dl=5),null}function ys(e){var n=e.expirationTime;return n>(e=e.childExpirationTime)?n:e}function ks(e){var n=qr();return Wr(99,ws.bind(null,e,n)),null}function ws(e,n){do{xs()}while(null!==Kl);if(0!=(48&Rl))throw Error(a(327));var t=e.finishedWork,i=e.finishedExpirationTime;if(null===t)return null;if(e.finishedWork=null,e.finishedExpirationTime=0,t===e.current)throw Error(a(177));e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90,e.nextKnownPendingLevel=0;var r=ys(t);if(e.firstPendingTime=r,i<=e.lastSuspendedTime?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:i<=e.firstSuspendedTime&&(e.firstSuspendedTime=i-1),i<=e.lastPingedTime&&(e.lastPingedTime=0),i<=e.lastExpiredTime&&(e.lastExpiredTime=0),e===Nl&&(Il=Nl=null,Ll=0),1<t.effectTag?null!==t.lastEffect?(t.lastEffect.nextEffect=t,r=t.firstEffect):r=t:r=t.firstEffect,null!==r){var o=Rl;Rl|=Pl,Sl.current=null,yt=Kn;var l=gt();if(ht(l)){if("selectionStart"in l)var s={start:l.selectionStart,end:l.selectionEnd};else e:{var c=(s=(s=l.ownerDocument)&&s.defaultView||window).getSelection&&s.getSelection();if(c&&0!==c.rangeCount){s=c.anchorNode;var u=c.anchorOffset,p=c.focusNode;c=c.focusOffset;try{s.nodeType,p.nodeType}catch(T){s=null;break e}var d=0,f=-1,m=-1,g=0,h=0,v=l,b=null;n:for(;;){for(var y;v!==s||0!==u&&3!==v.nodeType||(f=d+u),v!==p||0!==c&&3!==v.nodeType||(m=d+c),3===v.nodeType&&(d+=v.nodeValue.length),null!==(y=v.firstChild);)b=v,v=y;for(;;){if(v===l)break n;if(b===s&&++g===u&&(f=d),b===p&&++h===c&&(m=d),null!==(y=v.nextSibling))break;b=(v=b).parentNode}v=y}s=-1===f||-1===m?null:{start:f,end:m}}else s=null}s=s||{start:0,end:0}}else s=null;kt={activeElementDetached:null,focusedElem:l,selectionRange:s},Kn=!1,Gl=r;do{try{_s()}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);Gl=r;do{try{for(l=e,s=n;null!==Gl;){var k=Gl.effectTag;if(16&k&&Ue(Gl.stateNode,""),128&k){var w=Gl.alternate;if(null!==w){var _=w.ref;null!==_&&("function"==typeof _?_(null):_.current=null)}}switch(1038&k){case 2:fl(Gl),Gl.effectTag&=-3;break;case 6:fl(Gl),Gl.effectTag&=-3,vl(Gl.alternate,Gl);break;case 1024:Gl.effectTag&=-1025;break;case 1028:Gl.effectTag&=-1025,vl(Gl.alternate,Gl);break;case 4:vl(Gl.alternate,Gl);break;case 8:hl(l,u=Gl,s),pl(u)}Gl=Gl.nextEffect}}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);if(_=kt,w=gt(),k=_.focusedElem,s=_.selectionRange,w!==k&&k&&k.ownerDocument&&mt(k.ownerDocument.documentElement,k)){null!==s&&ht(k)&&(w=s.start,void 0===(_=s.end)&&(_=w),"selectionStart"in k?(k.selectionStart=w,k.selectionEnd=Math.min(_,k.value.length)):(_=(w=k.ownerDocument||document)&&w.defaultView||window).getSelection&&(_=_.getSelection(),u=k.textContent.length,l=Math.min(s.start,u),s=void 0===s.end?l:Math.min(s.end,u),!_.extend&&l>s&&(u=s,s=l,l=u),u=ft(k,l),p=ft(k,s),u&&p&&(1!==_.rangeCount||_.anchorNode!==u.node||_.anchorOffset!==u.offset||_.focusNode!==p.node||_.focusOffset!==p.offset)&&((w=w.createRange()).setStart(u.node,u.offset),_.removeAllRanges(),l>s?(_.addRange(w),_.extend(p.node,p.offset)):(w.setEnd(p.node,p.offset),_.addRange(w))))),w=[];for(_=k;_=_.parentNode;)1===_.nodeType&&w.push({element:_,left:_.scrollLeft,top:_.scrollTop});for("function"==typeof k.focus&&k.focus(),k=0;k<w.length;k++)(_=w[k]).element.scrollLeft=_.left,_.element.scrollTop=_.top}Kn=!!yt,kt=yt=null,e.current=t,Gl=r;do{try{for(k=e;null!==Gl;){var x=Gl.effectTag;if(36&x&&cl(k,Gl.alternate,Gl),128&x){w=void 0;var E=Gl.ref;if(null!==E){var S=Gl.stateNode;switch(Gl.tag){case 5:w=S;break;default:w=S}"function"==typeof E?E(w):E.current=w}}Gl=Gl.nextEffect}}catch(T){if(null===Gl)throw Error(a(330));Ts(Gl,T),Gl=Gl.nextEffect}}while(null!==Gl);Gl=null,Fr(),Rl=o}else e.current=t;if(Vl)Vl=!1,Kl=e,Yl=n;else for(Gl=r;null!==Gl;)n=Gl.nextEffect,Gl.nextEffect=null,Gl=n;if(0===(n=e.firstPendingTime)&&(Wl=null),1073741823===n?e===Jl?Xl++:(Xl=0,Jl=e):Xl=0,"function"==typeof Os&&Os(t.stateNode,i),os(e),ql)throw ql=!1,e=Hl,Hl=null,e;return 0!=(8&Rl)||Yr(),null}function _s(){for(;null!==Gl;){var e=Gl.effectTag;0!=(256&e)&&al(Gl.alternate,Gl),0==(512&e)||Vl||(Vl=!0,Vr(97,(function(){return xs(),null}))),Gl=Gl.nextEffect}}function xs(){if(90!==Yl){var e=97<Yl?97:Yl;return Yl=90,Wr(e,Es)}}function Es(){if(null===Kl)return!1;var e=Kl;if(Kl=null,0!=(48&Rl))throw Error(a(331));var n=Rl;for(Rl|=Pl,e=e.current.firstEffect;null!==e;){try{var t=e;if(0!=(512&t.effectTag))switch(t.tag){case 0:case 11:case 15:case 22:ll(5,t),sl(5,t)}}catch(i){if(null===e)throw Error(a(330));Ts(e,i)}t=e.nextEffect,e.nextEffect=null,e=t}return Rl=n,Yr(),!0}function Ss(e,n,t){fo(e,n=kl(e,n=tl(t,n),1073741823)),null!==(e=is(e,1073741823))&&os(e)}function Ts(e,n){if(3===e.tag)Ss(e,e,n);else for(var t=e.return;null!==t;){if(3===t.tag){Ss(t,e,n);break}if(1===t.tag){var i=t.stateNode;if("function"==typeof t.type.getDerivedStateFromError||"function"==typeof i.componentDidCatch&&(null===Wl||!Wl.has(i))){fo(t,e=wl(t,e=tl(n,e),1073741823)),null!==(t=is(t,1073741823))&&os(t);break}}t=t.return}}function Ps(e,n,t){var i=e.pingCache;null!==i&&i.delete(n),Nl===e&&Ll===t?Dl===Al||Dl===Ol&&1073741823===Ml&&Gr()-$l<500?us(e,Ll):Ul=!0:zs(e,t)&&(0!==(n=e.lastPingedTime)&&n<t||(e.lastPingedTime=t,os(e)))}function Cs(e,n){var t=e.stateNode;null!==t&&t.delete(n),0===(n=0)&&(n=ns(n=es(),e,null)),null!==(e=is(e,n))&&os(e)}_l=function(e,n,t){var i=n.expirationTime;if(null!==e){var r=n.pendingProps;if(e.memoizedProps!==r||hr.current)La=!0;else{if(i<t){switch(La=!1,n.tag){case 3:Ga(n),Na();break;case 5:if(Fo(n),4&n.mode&&1!==t&&r.hidden)return n.expirationTime=n.childExpirationTime=1,null;break;case 1:yr(n.type)&&xr(n);break;case 4:jo(n,n.stateNode.containerInfo);break;case 10:i=n.memoizedProps.value,r=n.type._context,fr(Zr,r._currentValue),r._currentValue=i;break;case 13:if(null!==n.memoizedState)return 0!==(i=n.child.childExpirationTime)&&i>=t?Ka(e,n,t):(fr(zo,1&zo.current),null!==(n=Ja(e,n,t))?n.sibling:null);fr(zo,1&zo.current);break;case 19:if(i=n.childExpirationTime>=t,0!=(64&e.effectTag)){if(i)return Xa(e,n,t);n.effectTag|=64}if(null!==(r=n.memoizedState)&&(r.rendering=null,r.tail=null),fr(zo,zo.current),!i)return null}return Ja(e,n,t)}La=!1}}else La=!1;switch(n.expirationTime=0,n.tag){case 2:if(i=n.type,null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),e=n.pendingProps,r=br(n,gr.current),ao(n,t),r=Jo(null,n,i,e,r,t),n.effectTag|=1,"object"==typeof r&&null!==r&&"function"==typeof r.render&&void 0===r.$$typeof){if(n.tag=1,n.memoizedState=null,n.updateQueue=null,yr(i)){var o=!0;xr(n)}else o=!1;n.memoizedState=null!==r.state&&void 0!==r.state?r.state:null,co(n);var l=i.getDerivedStateFromProps;"function"==typeof l&&yo(n,i,l,e),r.updater=ko,n.stateNode=r,r._reactInternalFiber=n,Eo(n,i,e,t),n=$a(null,n,i,!0,o,t)}else n.tag=0,Da(null,n,r,t),n=n.child;return n;case 16:e:{if(r=n.elementType,null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),e=n.pendingProps,function(e){if(-1===e._status){e._status=0;var n=e._ctor;n=n(),e._result=n,n.then((function(n){0===e._status&&(n=n.default,e._status=1,e._result=n)}),(function(n){0===e._status&&(e._status=2,e._result=n)}))}}(r),1!==r._status)throw r._result;switch(r=r._result,n.type=r,o=n.tag=function(e){if("function"==typeof e)return Is(e)?1:0;if(null!=e){if((e=e.$$typeof)===se)return 11;if(e===pe)return 14}return 2}(r),e=Jr(r,e),o){case 0:n=za(null,n,r,e,t);break e;case 1:n=Ua(null,n,r,e,t);break e;case 11:n=ja(null,n,r,e,t);break e;case 14:n=Ma(null,n,r,Jr(r.type,e),i,t);break e}throw Error(a(306,r,""))}return n;case 0:return i=n.type,r=n.pendingProps,za(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 1:return i=n.type,r=n.pendingProps,Ua(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 3:if(Ga(n),i=n.updateQueue,null===e||null===i)throw Error(a(282));if(i=n.pendingProps,r=null!==(r=n.memoizedState)?r.element:null,uo(e,n),go(n,i,null,t),(i=n.memoizedState.element)===r)Na(),n=Ja(e,n,t);else{if((r=n.stateNode.hydrate)&&(Sa=St(n.stateNode.containerInfo.firstChild),Ea=n,r=Ta=!0),r)for(t=Ao(n,null,i,t),n.child=t;t;)t.effectTag=-3&t.effectTag|1024,t=t.sibling;else Da(e,n,i,t),Na();n=n.child}return n;case 5:return Fo(n),null===e&&Oa(n),i=n.type,r=n.pendingProps,o=null!==e?e.memoizedProps:null,l=r.children,_t(i,r)?l=null:null!==o&&_t(i,o)&&(n.effectTag|=16),Ba(e,n),4&n.mode&&1!==t&&r.hidden?(n.expirationTime=n.childExpirationTime=1,n=null):(Da(e,n,l,t),n=n.child),n;case 6:return null===e&&Oa(n),null;case 13:return Ka(e,n,t);case 4:return jo(n,n.stateNode.containerInfo),i=n.pendingProps,null===e?n.child=Oo(n,null,i,t):Da(e,n,i,t),n.child;case 11:return i=n.type,r=n.pendingProps,ja(e,n,i,r=n.elementType===i?r:Jr(i,r),t);case 7:return Da(e,n,n.pendingProps,t),n.child;case 8:case 12:return Da(e,n,n.pendingProps.children,t),n.child;case 10:e:{i=n.type._context,r=n.pendingProps,l=n.memoizedProps,o=r.value;var s=n.type._context;if(fr(Zr,s._currentValue),s._currentValue=o,null!==l)if(s=l.value,0===(o=Ui(s,o)?0:0|("function"==typeof i._calculateChangedBits?i._calculateChangedBits(s,o):1073741823))){if(l.children===r.children&&!hr.current){n=Ja(e,n,t);break e}}else for(null!==(s=n.child)&&(s.return=n);null!==s;){var c=s.dependencies;if(null!==c){l=s.child;for(var u=c.firstContext;null!==u;){if(u.context===i&&0!=(u.observedBits&o)){1===s.tag&&((u=po(t,null)).tag=2,fo(s,u)),s.expirationTime<t&&(s.expirationTime=t),null!==(u=s.alternate)&&u.expirationTime<t&&(u.expirationTime=t),oo(s.return,t),c.expirationTime<t&&(c.expirationTime=t);break}u=u.next}}else l=10===s.tag&&s.type===n.type?null:s.child;if(null!==l)l.return=s;else for(l=s;null!==l;){if(l===n){l=null;break}if(null!==(s=l.sibling)){s.return=l.return,l=s;break}l=l.return}s=l}Da(e,n,r.children,t),n=n.child}return n;case 9:return r=n.type,i=(o=n.pendingProps).children,ao(n,t),i=i(r=lo(r,o.unstable_observedBits)),n.effectTag|=1,Da(e,n,i,t),n.child;case 14:return o=Jr(r=n.type,n.pendingProps),Ma(e,n,r,o=Jr(r.type,o),i,t);case 15:return Fa(e,n,n.type,n.pendingProps,i,t);case 17:return i=n.type,r=n.pendingProps,r=n.elementType===i?r:Jr(i,r),null!==e&&(e.alternate=null,n.alternate=null,n.effectTag|=2),n.tag=1,yr(i)?(e=!0,xr(n)):e=!1,ao(n,t),_o(n,i,r),Eo(n,i,r,t),$a(null,n,i,!0,e,t);case 19:return Xa(e,n,t)}throw Error(a(156,n.tag))};var Os=null,As=null;function Rs(e,n,t,i){this.tag=e,this.key=t,this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null,this.index=0,this.ref=null,this.pendingProps=n,this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null,this.mode=i,this.effectTag=0,this.lastEffect=this.firstEffect=this.nextEffect=null,this.childExpirationTime=this.expirationTime=0,this.alternate=null}function Ns(e,n,t,i){return new Rs(e,n,t,i)}function Is(e){return!(!(e=e.prototype)||!e.isReactComponent)}function Ls(e,n){var t=e.alternate;return null===t?((t=Ns(e.tag,n,e.key,e.mode)).elementType=e.elementType,t.type=e.type,t.stateNode=e.stateNode,t.alternate=e,e.alternate=t):(t.pendingProps=n,t.effectTag=0,t.nextEffect=null,t.firstEffect=null,t.lastEffect=null),t.childExpirationTime=e.childExpirationTime,t.expirationTime=e.expirationTime,t.child=e.child,t.memoizedProps=e.memoizedProps,t.memoizedState=e.memoizedState,t.updateQueue=e.updateQueue,n=e.dependencies,t.dependencies=null===n?null:{expirationTime:n.expirationTime,firstContext:n.firstContext,responders:n.responders},t.sibling=e.sibling,t.index=e.index,t.ref=e.ref,t}function Ds(e,n,t,i,r,o){var l=2;if(i=e,"function"==typeof e)Is(e)&&(l=1);else if("string"==typeof e)l=5;else e:switch(e){case te:return js(t.children,r,o,n);case le:l=8,r|=7;break;case ie:l=8,r|=1;break;case re:return(e=Ns(12,t,n,8|r)).elementType=re,e.type=re,e.expirationTime=o,e;case ce:return(e=Ns(13,t,n,r)).type=ce,e.elementType=ce,e.expirationTime=o,e;case ue:return(e=Ns(19,t,n,r)).elementType=ue,e.expirationTime=o,e;default:if("object"==typeof e&&null!==e)switch(e.$$typeof){case oe:l=10;break e;case ae:l=9;break e;case se:l=11;break e;case pe:l=14;break e;case de:l=16,i=null;break e;case fe:l=22;break e}throw Error(a(130,null==e?e:typeof e,""))}return(n=Ns(l,t,n,r)).elementType=e,n.type=i,n.expirationTime=o,n}function js(e,n,t,i){return(e=Ns(7,e,i,n)).expirationTime=t,e}function Ms(e,n,t){return(e=Ns(6,e,null,n)).expirationTime=t,e}function Fs(e,n,t){return(n=Ns(4,null!==e.children?e.children:[],e.key,n)).expirationTime=t,n.stateNode={containerInfo:e.containerInfo,pendingChildren:null,implementation:e.implementation},n}function Bs(e,n,t){this.tag=n,this.current=null,this.containerInfo=e,this.pingCache=this.pendingChildren=null,this.finishedExpirationTime=0,this.finishedWork=null,this.timeoutHandle=-1,this.pendingContext=this.context=null,this.hydrate=t,this.callbackNode=null,this.callbackPriority=90,this.lastExpiredTime=this.lastPingedTime=this.nextKnownPendingLevel=this.lastSuspendedTime=this.firstSuspendedTime=this.firstPendingTime=0}function zs(e,n){var t=e.firstSuspendedTime;return e=e.lastSuspendedTime,0!==t&&t>=n&&e<=n}function Us(e,n){var t=e.firstSuspendedTime,i=e.lastSuspendedTime;t<n&&(e.firstSuspendedTime=n),(i>n||0===t)&&(e.lastSuspendedTime=n),n<=e.lastPingedTime&&(e.lastPingedTime=0),n<=e.lastExpiredTime&&(e.lastExpiredTime=0)}function $s(e,n){n>e.firstPendingTime&&(e.firstPendingTime=n);var t=e.firstSuspendedTime;0!==t&&(n>=t?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:n>=e.lastSuspendedTime&&(e.lastSuspendedTime=n+1),n>e.nextKnownPendingLevel&&(e.nextKnownPendingLevel=n))}function Gs(e,n){var t=e.lastExpiredTime;(0===t||t>n)&&(e.lastExpiredTime=n)}function qs(e,n,t,i){var r=n.current,o=es(),l=vo.suspense;o=ns(o,r,l);e:if(t){n:{if(en(t=t._reactInternalFiber)!==t||1!==t.tag)throw Error(a(170));var s=t;do{switch(s.tag){case 3:s=s.stateNode.context;break n;case 1:if(yr(s.type)){s=s.stateNode.__reactInternalMemoizedMergedChildContext;break n}}s=s.return}while(null!==s);throw Error(a(171))}if(1===t.tag){var c=t.type;if(yr(c)){t=_r(t,c,s);break e}}t=s}else t=mr;return null===n.context?n.context=t:n.pendingContext=t,(n=po(o,l)).payload={element:e},null!==(i=void 0===i?null:i)&&(n.callback=i),fo(r,n),ts(r,o),o}function Hs(e){if(!(e=e.current).child)return null;switch(e.child.tag){case 5:default:return e.child.stateNode}}function Ws(e,n){null!==(e=e.memoizedState)&&null!==e.dehydrated&&e.retryTime<n&&(e.retryTime=n)}function Vs(e,n){Ws(e,n),(e=e.alternate)&&Ws(e,n)}function Ks(e,n,t){var i=new Bs(e,n,t=null!=t&&!0===t.hydrate),r=Ns(3,null,null,2===n?7:1===n?3:0);i.current=r,r.stateNode=i,co(r),e[At]=i.current,t&&0!==n&&function(e,n){var t=Ze(n);Cn.forEach((function(e){hn(e,n,t)})),On.forEach((function(e){hn(e,n,t)}))}(0,9===e.nodeType?e:e.ownerDocument),this._internalRoot=i}function Ys(e){return!(!e||1!==e.nodeType&&9!==e.nodeType&&11!==e.nodeType&&(8!==e.nodeType||" react-mount-point-unstable "!==e.nodeValue))}function Qs(e,n,t,i,r){var o=t._reactRootContainer;if(o){var a=o._internalRoot;if("function"==typeof r){var l=r;r=function(){var e=Hs(a);l.call(e)}}qs(n,a,e,r)}else{if(o=t._reactRootContainer=function(e,n){if(n||(n=!(!(n=e?9===e.nodeType?e.documentElement:e.firstChild:null)||1!==n.nodeType||!n.hasAttribute("data-reactroot"))),!n)for(var t;t=e.lastChild;)e.removeChild(t);return new Ks(e,0,n?{hydrate:!0}:void 0)}(t,i),a=o._internalRoot,"function"==typeof r){var s=r;r=function(){var e=Hs(a);s.call(e)}}cs((function(){qs(n,a,e,r)}))}return Hs(a)}function Xs(e,n,t){var i=3<arguments.length&&void 0!==arguments[3]?arguments[3]:null;return{$$typeof:ne,key:null==i?null:""+i,children:e,containerInfo:n,implementation:t}}function Js(e,n){var t=2<arguments.length&&void 0!==arguments[2]?arguments[2]:null;if(!Ys(n))throw Error(a(200));return Xs(e,n,null,t)}Ks.prototype.render=function(e){qs(e,this._internalRoot,null,null)},Ks.prototype.unmount=function(){var e=this._internalRoot,n=e.containerInfo;qs(null,e,null,(function(){n[At]=null}))},vn=function(e){if(13===e.tag){var n=Xr(es(),150,100);ts(e,n),Vs(e,n)}},bn=function(e){13===e.tag&&(ts(e,3),Vs(e,3))},yn=function(e){if(13===e.tag){var n=es();ts(e,n=ns(n,e,null)),Vs(e,n)}},C=function(e,n,t){switch(n){case"input":if(Se(e,t),n=t.name,"radio"===t.type&&null!=n){for(t=e;t.parentNode;)t=t.parentNode;for(t=t.querySelectorAll("input[name="+JSON.stringify(""+n)+'][type="radio"]'),n=0;n<t.length;n++){var i=t[n];if(i!==e&&i.form===e.form){var r=Lt(i);if(!r)throw Error(a(90));we(i),Se(i,r)}}}break;case"textarea":Ne(e,t);break;case"select":null!=(n=t.value)&&Oe(e,!!t.multiple,n,!1)}},L=ss,D=function(e,n,t,i,r){var o=Rl;Rl|=4;try{return Wr(98,e.bind(null,n,t,i,r))}finally{0===(Rl=o)&&Yr()}},j=function(){0==(49&Rl)&&(function(){if(null!==Ql){var e=Ql;Ql=null,e.forEach((function(e,n){Gs(n,e),os(n)})),Yr()}}(),xs())},M=function(e,n){var t=Rl;Rl|=2;try{return e(n)}finally{0===(Rl=t)&&Yr()}};var Zs={Events:[Nt,It,Lt,T,x,Ut,function(e){an(e,zt)},N,I,Zn,cn,xs,{current:!1}]};!function(e){var n=e.findFiberByHostInstance;(function(e){if("undefined"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__)return!1;var n=__REACT_DEVTOOLS_GLOBAL_HOOK__;if(n.isDisabled||!n.supportsFiber)return!0;try{var t=n.inject(e);Os=function(e){try{n.onCommitFiberRoot(t,e,void 0,64==(64&e.current.effectTag))}catch(i){}},As=function(e){try{n.onCommitFiberUnmount(t,e)}catch(i){}}}catch(i){}})(r({},e,{overrideHookState:null,overrideProps:null,setSuspenseHandler:null,scheduleUpdate:null,currentDispatcherRef:Q.ReactCurrentDispatcher,findHostInstanceByFiber:function(e){return null===(e=rn(e))?null:e.stateNode},findFiberByHostInstance:function(e){return n?n(e):null},findHostInstancesForRefresh:null,scheduleRefresh:null,scheduleRoot:null,setRefreshHandler:null,getCurrentFiber:null}))}({findFiberByHostInstance:Rt,bundleType:0,version:"16.14.0",rendererPackageName:"react-dom"}),n.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=Zs,n.createPortal=Js,n.findDOMNode=function(e){if(null==e)return null;if(1===e.nodeType)return e;var n=e._reactInternalFiber;if(void 0===n){if("function"==typeof e.render)throw Error(a(188));throw Error(a(268,Object.keys(e)))}return e=null===(e=rn(n))?null:e.stateNode},n.flushSync=function(e,n){if(0!=(48&Rl))throw Error(a(187));var t=Rl;Rl|=1;try{return Wr(99,e.bind(null,n))}finally{Rl=t,Yr()}},n.hydrate=function(e,n,t){if(!Ys(n))throw Error(a(200));return Qs(null,e,n,!0,t)},n.render=function(e,n,t){if(!Ys(n))throw Error(a(200));return Qs(null,e,n,!1,t)},n.unmountComponentAtNode=function(e){if(!Ys(e))throw Error(a(40));return!!e._reactRootContainer&&(cs((function(){Qs(null,null,e,!1,(function(){e._reactRootContainer=null,e[At]=null}))})),!0)},n.unstable_batchedUpdates=ss,n.unstable_createPortal=function(e,n){return Js(e,n,2<arguments.length&&void 0!==arguments[2]?arguments[2]:null)},n.unstable_renderSubtreeIntoContainer=function(e,n,t,i){if(!Ys(t))throw Error(a(200));if(null==e||void 0===e._reactInternalFiber)throw Error(a(38));return Qs(e,n,t,!1,i)},n.version="16.14.0"},function(e,n,t){"use strict";e.exports=t(42)},function(e,n,t){"use strict";var i,r,o,a,l;if("undefined"==typeof window||"function"!=typeof MessageChannel){var s=null,c=null,u=function(){if(null!==s)try{var e=n.unstable_now();s(!0,e),s=null}catch(t){throw setTimeout(u,0),t}},p=Date.now();n.unstable_now=function(){return Date.now()-p},i=function(e){null!==s?setTimeout(i,0,e):(s=e,setTimeout(u,0))},r=function(e,n){c=setTimeout(e,n)},o=function(){clearTimeout(c)},a=function(){return!1},l=n.unstable_forceFrameRate=function(){}}else{var d=window.performance,f=window.Date,m=window.setTimeout,g=window.clearTimeout;if("undefined"!=typeof console){var h=window.cancelAnimationFrame;"function"!=typeof window.requestAnimationFrame&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills"),"function"!=typeof h&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills")}if("object"==typeof d&&"function"==typeof d.now)n.unstable_now=function(){return d.now()};else{var v=f.now();n.unstable_now=function(){return f.now()-v}}var b=!1,y=null,k=-1,w=5,_=0;a=function(){return n.unstable_now()>=_},l=function(){},n.unstable_forceFrameRate=function(e){0>e||125<e?console.error("forceFrameRate takes a positive int between 0 and 125, forcing framerates higher than 125 fps is not unsupported"):w=0<e?Math.floor(1e3/e):5};var x=new MessageChannel,E=x.port2;x.port1.onmessage=function(){if(null!==y){var e=n.unstable_now();_=e+w;try{y(!0,e)?E.postMessage(null):(b=!1,y=null)}catch(t){throw E.postMessage(null),t}}else b=!1},i=function(e){y=e,b||(b=!0,E.postMessage(null))},r=function(e,t){k=m((function(){e(n.unstable_now())}),t)},o=function(){g(k),k=-1}}function S(e,n){var t=e.length;e.push(n);e:for(;;){var i=t-1>>>1,r=e[i];if(!(void 0!==r&&0<C(r,n)))break e;e[i]=n,e[t]=r,t=i}}function T(e){return void 0===(e=e[0])?null:e}function P(e){var n=e[0];if(void 0!==n){var t=e.pop();if(t!==n){e[0]=t;e:for(var i=0,r=e.length;i<r;){var o=2*(i+1)-1,a=e[o],l=o+1,s=e[l];if(void 0!==a&&0>C(a,t))void 0!==s&&0>C(s,a)?(e[i]=s,e[l]=t,i=l):(e[i]=a,e[o]=t,i=o);else{if(!(void 0!==s&&0>C(s,t)))break e;e[i]=s,e[l]=t,i=l}}}return n}return null}function C(e,n){var t=e.sortIndex-n.sortIndex;return 0!==t?t:e.id-n.id}var O=[],A=[],R=1,N=null,I=3,L=!1,D=!1,j=!1;function M(e){for(var n=T(A);null!==n;){if(null===n.callback)P(A);else{if(!(n.startTime<=e))break;P(A),n.sortIndex=n.expirationTime,S(O,n)}n=T(A)}}function F(e){if(j=!1,M(e),!D)if(null!==T(O))D=!0,i(B);else{var n=T(A);null!==n&&r(F,n.startTime-e)}}function B(e,t){D=!1,j&&(j=!1,o()),L=!0;var i=I;try{for(M(t),N=T(O);null!==N&&(!(N.expirationTime>t)||e&&!a());){var l=N.callback;if(null!==l){N.callback=null,I=N.priorityLevel;var s=l(N.expirationTime<=t);t=n.unstable_now(),"function"==typeof s?N.callback=s:N===T(O)&&P(O),M(t)}else P(O);N=T(O)}if(null!==N)var c=!0;else{var u=T(A);null!==u&&r(F,u.startTime-t),c=!1}return c}finally{N=null,I=i,L=!1}}function z(e){switch(e){case 1:return-1;case 2:return 250;case 5:return 1073741823;case 4:return 1e4;default:return 5e3}}var U=l;n.unstable_IdlePriority=5,n.unstable_ImmediatePriority=1,n.unstable_LowPriority=4,n.unstable_NormalPriority=3,n.unstable_Profiling=null,n.unstable_UserBlockingPriority=2,n.unstable_cancelCallback=function(e){e.callback=null},n.unstable_continueExecution=function(){D||L||(D=!0,i(B))},n.unstable_getCurrentPriorityLevel=function(){return I},n.unstable_getFirstCallbackNode=function(){return T(O)},n.unstable_next=function(e){switch(I){case 1:case 2:case 3:var n=3;break;default:n=I}var t=I;I=n;try{return e()}finally{I=t}},n.unstable_pauseExecution=function(){},n.unstable_requestPaint=U,n.unstable_runWithPriority=function(e,n){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var t=I;I=e;try{return n()}finally{I=t}},n.unstable_scheduleCallback=function(e,t,a){var l=n.unstable_now();if("object"==typeof a&&null!==a){var s=a.delay;s="number"==typeof s&&0<s?l+s:l,a="number"==typeof a.timeout?a.timeout:z(e)}else a=z(e),s=l;return e={id:R++,callback:t,priorityLevel:e,startTime:s,expirationTime:a=s+a,sortIndex:-1},s>l?(e.sortIndex=s,S(A,e),null===T(O)&&e===T(A)&&(j?o():j=!0,r(F,s-l))):(e.sortIndex=a,S(O,e),D||L||(D=!0,i(B))),e},n.unstable_shouldYield=function(){var e=n.unstable_now();M(e);var t=T(O);return t!==N&&null!==N&&null!==t&&null!==t.callback&&t.startTime<=e&&t.expirationTime<N.expirationTime||a()},n.unstable_wrapCallback=function(e){var n=I;return function(){var t=I;I=n;try{return e.apply(this,arguments)}finally{I=t}}}},function(e,n,t){"use strict";var i=t(44);function r(){}function o(){}o.resetWarningCache=r,e.exports=function(){function e(e,n,t,r,o,a){if(a!==i){var l=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw l.name="Invariant Violation",l}}function n(){return e}e.isRequired=e;var t={array:e,bigint:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:n,element:e,elementType:e,instanceOf:n,node:e,objectOf:n,oneOf:n,oneOfType:n,shape:n,exact:n,checkPropTypes:o,resetWarningCache:r};return t.PropTypes=t,t}},function(e,n,t){"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},function(e,n){e.exports=Array.isArray||function(e){return"[object Array]"==Object.prototype.toString.call(e)}},function(e,n,t){"use strict";var i="function"==typeof Symbol&&Symbol.for,r=i?Symbol.for("react.element"):60103,o=i?Symbol.for("react.portal"):60106,a=i?Symbol.for("react.fragment"):60107,l=i?Symbol.for("react.strict_mode"):60108,s=i?Symbol.for("react.profiler"):60114,c=i?Symbol.for("react.provider"):60109,u=i?Symbol.for("react.context"):60110,p=i?Symbol.for("react.async_mode"):60111,d=i?Symbol.for("react.concurrent_mode"):60111,f=i?Symbol.for("react.forward_ref"):60112,m=i?Symbol.for("react.suspense"):60113,g=i?Symbol.for("react.suspense_list"):60120,h=i?Symbol.for("react.memo"):60115,v=i?Symbol.for("react.lazy"):60116,b=i?Symbol.for("react.block"):60121,y=i?Symbol.for("react.fundamental"):60117,k=i?Symbol.for("react.responder"):60118,w=i?Symbol.for("react.scope"):60119;function _(e){if("object"==typeof e&&null!==e){var n=e.$$typeof;switch(n){case r:switch(e=e.type){case p:case d:case a:case s:case l:case m:return e;default:switch(e=e&&e.$$typeof){case u:case f:case v:case h:case c:return e;default:return n}}case o:return n}}}function x(e){return _(e)===d}n.AsyncMode=p,n.ConcurrentMode=d,n.ContextConsumer=u,n.ContextProvider=c,n.Element=r,n.ForwardRef=f,n.Fragment=a,n.Lazy=v,n.Memo=h,n.Portal=o,n.Profiler=s,n.StrictMode=l,n.Suspense=m,n.isAsyncMode=function(e){return x(e)||_(e)===p},n.isConcurrentMode=x,n.isContextConsumer=function(e){return _(e)===u},n.isContextProvider=function(e){return _(e)===c},n.isElement=function(e){return"object"==typeof e&&null!==e&&e.$$typeof===r},n.isForwardRef=function(e){return _(e)===f},n.isFragment=function(e){return _(e)===a},n.isLazy=function(e){return _(e)===v},n.isMemo=function(e){return _(e)===h},n.isPortal=function(e){return _(e)===o},n.isProfiler=function(e){return _(e)===s},n.isStrictMode=function(e){return _(e)===l},n.isSuspense=function(e){return _(e)===m},n.isValidElementType=function(e){return"string"==typeof e||"function"==typeof e||e===a||e===d||e===s||e===l||e===m||e===g||"object"==typeof e&&null!==e&&(e.$$typeof===v||e.$$typeof===h||e.$$typeof===c||e.$$typeof===u||e.$$typeof===f||e.$$typeof===y||e.$$typeof===k||e.$$typeof===w||e.$$typeof===b)},n.typeOf=_},,function(e,n,t){const i=t(49),r=t(50),o=new Set;function a(e){void 0===e?e=Object.keys(i.languages).filter((e=>"meta"!=e)):Array.isArray(e)||(e=[e]);const n=[...o,...Object.keys(Prism.languages)];r(i,e,n).load((e=>{if(!(e in i.languages))return void(a.silent||console.warn("Language does not exist: "+e));const n="./prism-"+e;delete t.c[t(19).resolve(n)],delete Prism.languages[e],t(19)(n),o.add(e)}))}a.silent=!1,e.exports=a},function(e,n,t){e.exports&&(e.exports={core:{meta:{path:"components/prism-core.js",option:"mandatory"},core:"Core"},themes:{meta:{path:"themes/{id}.css",link:"index.html?theme={id}",exclusive:!0},prism:{title:"Default",option:"default"},"prism-dark":"Dark","prism-funky":"Funky","prism-okaidia":{title:"Okaidia",owner:"ocodia"},"prism-twilight":{title:"Twilight",owner:"remybach"},"prism-coy":{title:"Coy",owner:"tshedor"},"prism-solarizedlight":{title:"Solarized Light",owner:"hectormatos2011 "},"prism-tomorrow":{title:"Tomorrow Night",owner:"Rosey"}},languages:{meta:{path:"components/prism-{id}",noCSS:!0,examplesPath:"examples/prism-{id}",addCheckAll:!0},markup:{title:"Markup",alias:["html","xml","svg","mathml","ssml","atom","rss"],aliasTitles:{html:"HTML",xml:"XML",svg:"SVG",mathml:"MathML",ssml:"SSML",atom:"Atom",rss:"RSS"},option:"default"},css:{title:"CSS",option:"default",modify:"markup"},clike:{title:"C-like",option:"default"},javascript:{title:"JavaScript",require:"clike",modify:"markup",optional:"regex",alias:"js",option:"default"},abap:{title:"ABAP",owner:"dellagustin"},abnf:{title:"ABNF",owner:"RunDevelopment"},actionscript:{title:"ActionScript",require:"javascript",modify:"markup",owner:"Golmote"},ada:{title:"Ada",owner:"Lucretia"},agda:{title:"Agda",owner:"xy-ren"},al:{title:"AL",owner:"RunDevelopment"},antlr4:{title:"ANTLR4",alias:"g4",owner:"RunDevelopment"},apacheconf:{title:"Apache Configuration",owner:"GuiTeK"},apex:{title:"Apex",require:["clike","sql"],owner:"RunDevelopment"},apl:{title:"APL",owner:"ngn"},applescript:{title:"AppleScript",owner:"Golmote"},aql:{title:"AQL",owner:"RunDevelopment"},arduino:{title:"Arduino",require:"cpp",owner:"dkern"},arff:{title:"ARFF",owner:"Golmote"},asciidoc:{alias:"adoc",title:"AsciiDoc",owner:"Golmote"},aspnet:{title:"ASP.NET (C#)",require:["markup","csharp"],owner:"nauzilus"},asm6502:{title:"6502 Assembly",owner:"kzurawel"},autohotkey:{title:"AutoHotkey",owner:"aviaryan"},autoit:{title:"AutoIt",owner:"Golmote"},avisynth:{title:"AviSynth",alias:"avs",owner:"Zinfidel"},"avro-idl":{title:"Avro IDL",alias:"avdl",owner:"RunDevelopment"},bash:{title:"Bash",alias:"shell",aliasTitles:{shell:"Shell"},owner:"zeitgeist87"},basic:{title:"BASIC",owner:"Golmote"},batch:{title:"Batch",owner:"Golmote"},bbcode:{title:"BBcode",alias:"shortcode",aliasTitles:{shortcode:"Shortcode"},owner:"RunDevelopment"},bicep:{title:"Bicep",owner:"johnnyreilly"},birb:{title:"Birb",require:"clike",owner:"Calamity210"},bison:{title:"Bison",require:"c",owner:"Golmote"},bnf:{title:"BNF",alias:"rbnf",aliasTitles:{rbnf:"RBNF"},owner:"RunDevelopment"},brainfuck:{title:"Brainfuck",owner:"Golmote"},brightscript:{title:"BrightScript",owner:"RunDevelopment"},bro:{title:"Bro",owner:"wayward710"},bsl:{title:"BSL (1C:Enterprise)",alias:"oscript",aliasTitles:{oscript:"OneScript"},owner:"Diversus23"},c:{title:"C",require:"clike",owner:"zeitgeist87"},csharp:{title:"C#",require:"clike",alias:["cs","dotnet"],owner:"mvalipour"},cpp:{title:"C++",require:"c",owner:"zeitgeist87"},cfscript:{title:"CFScript",require:"clike",alias:"cfc",owner:"mjclemente"},chaiscript:{title:"ChaiScript",require:["clike","cpp"],owner:"RunDevelopment"},cil:{title:"CIL",owner:"sbrl"},clojure:{title:"Clojure",owner:"troglotit"},cmake:{title:"CMake",owner:"mjrogozinski"},cobol:{title:"COBOL",owner:"RunDevelopment"},coffeescript:{title:"CoffeeScript",require:"javascript",alias:"coffee",owner:"R-osey"},concurnas:{title:"Concurnas",alias:"conc",owner:"jasontatton"},csp:{title:"Content-Security-Policy",owner:"ScottHelme"},coq:{title:"Coq",owner:"RunDevelopment"},crystal:{title:"Crystal",require:"ruby",owner:"MakeNowJust"},"css-extras":{title:"CSS Extras",require:"css",modify:"css",owner:"milesj"},csv:{title:"CSV",owner:"RunDevelopment"},cypher:{title:"Cypher",owner:"RunDevelopment"},d:{title:"D",require:"clike",owner:"Golmote"},dart:{title:"Dart",require:"clike",owner:"Golmote"},dataweave:{title:"DataWeave",owner:"machaval"},dax:{title:"DAX",owner:"peterbud"},dhall:{title:"Dhall",owner:"RunDevelopment"},diff:{title:"Diff",owner:"uranusjr"},django:{title:"Django/Jinja2",require:"markup-templating",alias:"jinja2",owner:"romanvm"},"dns-zone-file":{title:"DNS zone file",owner:"RunDevelopment",alias:"dns-zone"},docker:{title:"Docker",alias:"dockerfile",owner:"JustinBeckwith"},dot:{title:"DOT (Graphviz)",alias:"gv",optional:"markup",owner:"RunDevelopment"},ebnf:{title:"EBNF",owner:"RunDevelopment"},editorconfig:{title:"EditorConfig",owner:"osipxd"},eiffel:{title:"Eiffel",owner:"Conaclos"},ejs:{title:"EJS",require:["javascript","markup-templating"],owner:"RunDevelopment",alias:"eta",aliasTitles:{eta:"Eta"}},elixir:{title:"Elixir",owner:"Golmote"},elm:{title:"Elm",owner:"zwilias"},etlua:{title:"Embedded Lua templating",require:["lua","markup-templating"],owner:"RunDevelopment"},erb:{title:"ERB",require:["ruby","markup-templating"],owner:"Golmote"},erlang:{title:"Erlang",owner:"Golmote"},"excel-formula":{title:"Excel Formula",alias:["xlsx","xls"],owner:"RunDevelopment"},fsharp:{title:"F#",require:"clike",owner:"simonreynolds7"},factor:{title:"Factor",owner:"catb0t"},false:{title:"False",owner:"edukisto"},"firestore-security-rules":{title:"Firestore security rules",require:"clike",owner:"RunDevelopment"},flow:{title:"Flow",require:"javascript",owner:"Golmote"},fortran:{title:"Fortran",owner:"Golmote"},ftl:{title:"FreeMarker Template Language",require:"markup-templating",owner:"RunDevelopment"},gml:{title:"GameMaker Language",alias:"gamemakerlanguage",require:"clike",owner:"LiarOnce"},gap:{title:"GAP (CAS)",owner:"RunDevelopment"},gcode:{title:"G-code",owner:"RunDevelopment"},gdscript:{title:"GDScript",owner:"RunDevelopment"},gedcom:{title:"GEDCOM",owner:"Golmote"},gherkin:{title:"Gherkin",owner:"hason"},git:{title:"Git",owner:"lgiraudel"},glsl:{title:"GLSL",require:"c",owner:"Golmote"},gn:{title:"GN",alias:"gni",owner:"RunDevelopment"},go:{title:"Go",require:"clike",owner:"arnehormann"},graphql:{title:"GraphQL",optional:"markdown",owner:"Golmote"},groovy:{title:"Groovy",require:"clike",owner:"robfletcher"},haml:{title:"Haml",require:"ruby",optional:["css","css-extras","coffeescript","erb","javascript","less","markdown","scss","textile"],owner:"Golmote"},handlebars:{title:"Handlebars",require:"markup-templating",alias:"hbs",owner:"Golmote"},haskell:{title:"Haskell",alias:"hs",owner:"bholst"},haxe:{title:"Haxe",require:"clike",owner:"Golmote"},hcl:{title:"HCL",owner:"outsideris"},hlsl:{title:"HLSL",require:"c",owner:"RunDevelopment"},hoon:{title:"Hoon",owner:"matildepark"},http:{title:"HTTP",optional:["css","javascript","json","markup","uri"],owner:"danielgtaylor"},hpkp:{title:"HTTP Public-Key-Pins",owner:"ScottHelme"},hsts:{title:"HTTP Strict-Transport-Security",owner:"ScottHelme"},ichigojam:{title:"IchigoJam",owner:"BlueCocoa"},icon:{title:"Icon",owner:"Golmote"},"icu-message-format":{title:"ICU Message Format",owner:"RunDevelopment"},idris:{title:"Idris",alias:"idr",owner:"KeenS",require:"haskell"},ignore:{title:".ignore",owner:"osipxd",alias:["gitignore","hgignore","npmignore"],aliasTitles:{gitignore:".gitignore",hgignore:".hgignore",npmignore:".npmignore"}},inform7:{title:"Inform 7",owner:"Golmote"},ini:{title:"Ini",owner:"aviaryan"},io:{title:"Io",owner:"AlesTsurko"},j:{title:"J",owner:"Golmote"},java:{title:"Java",require:"clike",owner:"sherblot"},javadoc:{title:"JavaDoc",require:["markup","java","javadoclike"],modify:"java",optional:"scala",owner:"RunDevelopment"},javadoclike:{title:"JavaDoc-like",modify:["java","javascript","php"],owner:"RunDevelopment"},javastacktrace:{title:"Java stack trace",owner:"RunDevelopment"},jexl:{title:"Jexl",owner:"czosel"},jolie:{title:"Jolie",require:"clike",owner:"thesave"},jq:{title:"JQ",owner:"RunDevelopment"},jsdoc:{title:"JSDoc",require:["javascript","javadoclike","typescript"],modify:"javascript",optional:["actionscript","coffeescript"],owner:"RunDevelopment"},"js-extras":{title:"JS Extras",require:"javascript",modify:"javascript",optional:["actionscript","coffeescript","flow","n4js","typescript"],owner:"RunDevelopment"},json:{title:"JSON",alias:"webmanifest",aliasTitles:{webmanifest:"Web App Manifest"},owner:"CupOfTea696"},json5:{title:"JSON5",require:"json",owner:"RunDevelopment"},jsonp:{title:"JSONP",require:"json",owner:"RunDevelopment"},jsstacktrace:{title:"JS stack trace",owner:"sbrl"},"js-templates":{title:"JS Templates",require:"javascript",modify:"javascript",optional:["css","css-extras","graphql","markdown","markup","sql"],owner:"RunDevelopment"},julia:{title:"Julia",owner:"cdagnino"},keyman:{title:"Keyman",owner:"mcdurdin"},kotlin:{title:"Kotlin",alias:["kt","kts"],aliasTitles:{kts:"Kotlin Script"},require:"clike",owner:"Golmote"},kumir:{title:"KuMir (\u041a\u0443\u041c\u0438\u0440)",alias:"kum",owner:"edukisto"},kusto:{title:"Kusto",owner:"RunDevelopment"},latex:{title:"LaTeX",alias:["tex","context"],aliasTitles:{tex:"TeX",context:"ConTeXt"},owner:"japborst"},latte:{title:"Latte",require:["clike","markup-templating","php"],owner:"nette"},less:{title:"Less",require:"css",optional:"css-extras",owner:"Golmote"},lilypond:{title:"LilyPond",require:"scheme",alias:"ly",owner:"RunDevelopment"},liquid:{title:"Liquid",require:"markup-templating",owner:"cinhtau"},lisp:{title:"Lisp",alias:["emacs","elisp","emacs-lisp"],owner:"JuanCaicedo"},livescript:{title:"LiveScript",owner:"Golmote"},llvm:{title:"LLVM IR",owner:"porglezomp"},log:{title:"Log file",optional:"javastacktrace",owner:"RunDevelopment"},lolcode:{title:"LOLCODE",owner:"Golmote"},lua:{title:"Lua",owner:"Golmote"},magma:{title:"Magma (CAS)",owner:"RunDevelopment"},makefile:{title:"Makefile",owner:"Golmote"},markdown:{title:"Markdown",require:"markup",optional:"yaml",alias:"md",owner:"Golmote"},"markup-templating":{title:"Markup templating",require:"markup",owner:"Golmote"},matlab:{title:"MATLAB",owner:"Golmote"},maxscript:{title:"MAXScript",owner:"RunDevelopment"},mel:{title:"MEL",owner:"Golmote"},mermaid:{title:"Mermaid",owner:"RunDevelopment"},mizar:{title:"Mizar",owner:"Golmote"},mongodb:{title:"MongoDB",owner:"airs0urce",require:"javascript"},monkey:{title:"Monkey",owner:"Golmote"},moonscript:{title:"MoonScript",alias:"moon",owner:"RunDevelopment"},n1ql:{title:"N1QL",owner:"TMWilds"},n4js:{title:"N4JS",require:"javascript",optional:"jsdoc",alias:"n4jsd",owner:"bsmith-n4"},"nand2tetris-hdl":{title:"Nand To Tetris HDL",owner:"stephanmax"},naniscript:{title:"Naninovel Script",owner:"Elringus",alias:"nani"},nasm:{title:"NASM",owner:"rbmj"},neon:{title:"NEON",owner:"nette"},nevod:{title:"Nevod",owner:"nezaboodka"},nginx:{title:"nginx",owner:"volado"},nim:{title:"Nim",owner:"Golmote"},nix:{title:"Nix",owner:"Golmote"},nsis:{title:"NSIS",owner:"idleberg"},objectivec:{title:"Objective-C",require:"c",alias:"objc",owner:"uranusjr"},ocaml:{title:"OCaml",owner:"Golmote"},opencl:{title:"OpenCL",require:"c",modify:["c","cpp"],owner:"Milania1"},openqasm:{title:"OpenQasm",alias:"qasm",owner:"RunDevelopment"},oz:{title:"Oz",owner:"Golmote"},parigp:{title:"PARI/GP",owner:"Golmote"},parser:{title:"Parser",require:"markup",owner:"Golmote"},pascal:{title:"Pascal",alias:"objectpascal",aliasTitles:{objectpascal:"Object Pascal"},owner:"Golmote"},pascaligo:{title:"Pascaligo",owner:"DefinitelyNotAGoat"},psl:{title:"PATROL Scripting Language",owner:"bertysentry"},pcaxis:{title:"PC-Axis",alias:"px",owner:"RunDevelopment"},peoplecode:{title:"PeopleCode",alias:"pcode",owner:"RunDevelopment"},perl:{title:"Perl",owner:"Golmote"},php:{title:"PHP",require:"markup-templating",owner:"milesj"},phpdoc:{title:"PHPDoc",require:["php","javadoclike"],modify:"php",owner:"RunDevelopment"},"php-extras":{title:"PHP Extras",require:"php",modify:"php",owner:"milesj"},plsql:{title:"PL/SQL",require:"sql",owner:"Golmote"},powerquery:{title:"PowerQuery",alias:["pq","mscript"],owner:"peterbud"},powershell:{title:"PowerShell",owner:"nauzilus"},processing:{title:"Processing",require:"clike",owner:"Golmote"},prolog:{title:"Prolog",owner:"Golmote"},promql:{title:"PromQL",owner:"arendjr"},properties:{title:".properties",owner:"Golmote"},protobuf:{title:"Protocol Buffers",require:"clike",owner:"just-boris"},pug:{title:"Pug",require:["markup","javascript"],optional:["coffeescript","ejs","handlebars","less","livescript","markdown","scss","stylus","twig"],owner:"Golmote"},puppet:{title:"Puppet",owner:"Golmote"},pure:{title:"Pure",optional:["c","cpp","fortran"],owner:"Golmote"},purebasic:{title:"PureBasic",require:"clike",alias:"pbfasm",owner:"HeX0R101"},purescript:{title:"PureScript",require:"haskell",alias:"purs",owner:"sriharshachilakapati"},python:{title:"Python",alias:"py",owner:"multipetros"},qsharp:{title:"Q#",require:"clike",alias:"qs",owner:"fedonman"},q:{title:"Q (kdb+ database)",owner:"Golmote"},qml:{title:"QML",require:"javascript",owner:"RunDevelopment"},qore:{title:"Qore",require:"clike",owner:"temnroegg"},r:{title:"R",owner:"Golmote"},racket:{title:"Racket",require:"scheme",alias:"rkt",owner:"RunDevelopment"},cshtml:{title:"Razor C#",alias:"razor",require:["markup","csharp"],optional:["css","css-extras","javascript","js-extras"],owner:"RunDevelopment"},jsx:{title:"React JSX",require:["markup","javascript"],optional:["jsdoc","js-extras","js-templates"],owner:"vkbansal"},tsx:{title:"React TSX",require:["jsx","typescript"]},reason:{title:"Reason",require:"clike",owner:"Golmote"},regex:{title:"Regex",owner:"RunDevelopment"},rego:{title:"Rego",owner:"JordanSh"},renpy:{title:"Ren'py",alias:"rpy",owner:"HyuchiaDiego"},rest:{title:"reST (reStructuredText)",owner:"Golmote"},rip:{title:"Rip",owner:"ravinggenius"},roboconf:{title:"Roboconf",owner:"Golmote"},robotframework:{title:"Robot Framework",alias:"robot",owner:"RunDevelopment"},ruby:{title:"Ruby",require:"clike",alias:"rb",owner:"samflores"},rust:{title:"Rust",owner:"Golmote"},sas:{title:"SAS",optional:["groovy","lua","sql"],owner:"Golmote"},sass:{title:"Sass (Sass)",require:"css",optional:"css-extras",owner:"Golmote"},scss:{title:"Sass (Scss)",require:"css",optional:"css-extras",owner:"MoOx"},scala:{title:"Scala",require:"java",owner:"jozic"},scheme:{title:"Scheme",owner:"bacchus123"},"shell-session":{title:"Shell session",require:"bash",alias:["sh-session","shellsession"],owner:"RunDevelopment"},smali:{title:"Smali",owner:"RunDevelopment"},smalltalk:{title:"Smalltalk",owner:"Golmote"},smarty:{title:"Smarty",require:"markup-templating",owner:"Golmote"},sml:{title:"SML",alias:"smlnj",aliasTitles:{smlnj:"SML/NJ"},owner:"RunDevelopment"},solidity:{title:"Solidity (Ethereum)",alias:"sol",require:"clike",owner:"glachaud"},"solution-file":{title:"Solution file",alias:"sln",owner:"RunDevelopment"},soy:{title:"Soy (Closure Template)",require:"markup-templating",owner:"Golmote"},sparql:{title:"SPARQL",require:"turtle",owner:"Triply-Dev",alias:"rq"},"splunk-spl":{title:"Splunk SPL",owner:"RunDevelopment"},sqf:{title:"SQF: Status Quo Function (Arma 3)",require:"clike",owner:"RunDevelopment"},sql:{title:"SQL",owner:"multipetros"},squirrel:{title:"Squirrel",require:"clike",owner:"RunDevelopment"},stan:{title:"Stan",owner:"RunDevelopment"},iecst:{title:"Structured Text (IEC 61131-3)",owner:"serhioromano"},stylus:{title:"Stylus",owner:"vkbansal"},swift:{title:"Swift",owner:"chrischares"},systemd:{title:"Systemd configuration file",owner:"RunDevelopment"},"t4-templating":{title:"T4 templating",owner:"RunDevelopment"},"t4-cs":{title:"T4 Text Templates (C#)",require:["t4-templating","csharp"],alias:"t4",owner:"RunDevelopment"},"t4-vb":{title:"T4 Text Templates (VB)",require:["t4-templating","vbnet"],owner:"RunDevelopment"},tap:{title:"TAP",owner:"isaacs",require:"yaml"},tcl:{title:"Tcl",owner:"PeterChaplin"},tt2:{title:"Template Toolkit 2",require:["clike","markup-templating"],owner:"gflohr"},textile:{title:"Textile",require:"markup",optional:"css",owner:"Golmote"},toml:{title:"TOML",owner:"RunDevelopment"},turtle:{title:"Turtle",alias:"trig",aliasTitles:{trig:"TriG"},owner:"jakubklimek"},twig:{title:"Twig",require:"markup",owner:"brandonkelly"},typescript:{title:"TypeScript",require:"javascript",optional:"js-templates",alias:"ts",owner:"vkbansal"},typoscript:{title:"TypoScript",alias:"tsconfig",aliasTitles:{tsconfig:"TSConfig"},owner:"dkern"},unrealscript:{title:"UnrealScript",alias:["uscript","uc"],owner:"RunDevelopment"},uri:{title:"URI",alias:"url",aliasTitles:{url:"URL"},owner:"RunDevelopment"},v:{title:"V",require:"clike",owner:"taggon"},vala:{title:"Vala",require:"clike",optional:"regex",owner:"TemplarVolk"},vbnet:{title:"VB.Net",require:"basic",owner:"Bigsby"},velocity:{title:"Velocity",require:"markup",owner:"Golmote"},verilog:{title:"Verilog",owner:"a-rey"},vhdl:{title:"VHDL",owner:"a-rey"},vim:{title:"vim",owner:"westonganger"},"visual-basic":{title:"Visual Basic",alias:["vb","vba"],aliasTitles:{vba:"VBA"},owner:"Golmote"},warpscript:{title:"WarpScript",owner:"RunDevelopment"},wasm:{title:"WebAssembly",owner:"Golmote"},wiki:{title:"Wiki markup",require:"markup",owner:"Golmote"},wolfram:{title:"Wolfram language",alias:["mathematica","nb","wl"],aliasTitles:{mathematica:"Mathematica",nb:"Mathematica Notebook"},owner:"msollami"},wren:{title:"Wren",owner:"clsource"},xeora:{title:"Xeora",require:"markup",alias:"xeoracube",aliasTitles:{xeoracube:"XeoraCube"},owner:"freakmaxi"},"xml-doc":{title:"XML doc (.net)",require:"markup",modify:["csharp","fsharp","vbnet"],owner:"RunDevelopment"},xojo:{title:"Xojo (REALbasic)",owner:"Golmote"},xquery:{title:"XQuery",require:"markup",owner:"Golmote"},yaml:{title:"YAML",alias:"yml",owner:"hason"},yang:{title:"YANG",owner:"RunDevelopment"},zig:{title:"Zig",owner:"RunDevelopment"}},plugins:{meta:{path:"plugins/{id}/prism-{id}",link:"plugins/{id}/"},"line-highlight":{title:"Line Highlight",description:"Highlights specific lines and/or line ranges."},"line-numbers":{title:"Line Numbers",description:"Line number at the beginning of code lines.",owner:"kuba-kubula"},"show-invisibles":{title:"Show Invisibles",description:"Show hidden characters such as tabs and line breaks.",optional:["autolinker","data-uri-highlight"]},autolinker:{title:"Autolinker",description:"Converts URLs and emails in code to clickable links. Parses Markdown links in comments."},wpd:{title:"WebPlatform Docs",description:'Makes tokens link to <a href="https://webplatform.github.io/docs/">WebPlatform.org documentation</a>. The links open in a new tab.'},"custom-class":{title:"Custom Class",description:"This plugin allows you to prefix Prism's default classes (<code>.comment</code> can become <code>.namespace--comment</code>) or replace them with your defined ones (like <code>.editor__comment</code>). You can even add new classes.",owner:"dvkndn",noCSS:!0},"file-highlight":{title:"File Highlight",description:"Fetch external files and highlight them with Prism. Used on the Prism website itself.",noCSS:!0},"show-language":{title:"Show Language",description:"Display the highlighted language in code blocks (inline code does not show the label).",owner:"nauzilus",noCSS:!0,require:"toolbar"},"jsonp-highlight":{title:"JSONP Highlight",description:"Fetch content with JSONP and highlight some interesting content (e.g. GitHub/Gists or Bitbucket API).",noCSS:!0,owner:"nauzilus"},"highlight-keywords":{title:"Highlight Keywords",description:"Adds special CSS classes for each keyword for fine-grained highlighting.",owner:"vkbansal",noCSS:!0},"remove-initial-line-feed":{title:"Remove initial line feed",description:"Removes the initial line feed in code blocks.",owner:"Golmote",noCSS:!0},"inline-color":{title:"Inline color",description:"Adds a small inline preview for colors in style sheets.",require:"css-extras",owner:"RunDevelopment"},previewers:{title:"Previewers",description:"Previewers for angles, colors, gradients, easing and time.",require:"css-extras",owner:"Golmote"},autoloader:{title:"Autoloader",description:"Automatically loads the needed languages to highlight the code blocks.",owner:"Golmote",noCSS:!0},"keep-markup":{title:"Keep Markup",description:"Prevents custom markup from being dropped out during highlighting.",owner:"Golmote",optional:"normalize-whitespace",noCSS:!0},"command-line":{title:"Command Line",description:"Display a command line with a prompt and, optionally, the output/response from the commands.",owner:"chriswells0"},"unescaped-markup":{title:"Unescaped Markup",description:"Write markup without having to escape anything."},"normalize-whitespace":{title:"Normalize Whitespace",description:"Supports multiple operations to normalize whitespace in code blocks.",owner:"zeitgeist87",optional:"unescaped-markup",noCSS:!0},"data-uri-highlight":{title:"Data-URI Highlight",description:"Highlights data-URI contents.",owner:"Golmote",noCSS:!0},toolbar:{title:"Toolbar",description:"Attach a toolbar for plugins to easily register buttons on the top of a code block.",owner:"mAAdhaTTah"},"copy-to-clipboard":{title:"Copy to Clipboard Button",description:"Add a button that copies the code block to the clipboard when clicked.",owner:"mAAdhaTTah",require:"toolbar",noCSS:!0},"download-button":{title:"Download Button",description:"A button in the toolbar of a code block adding a convenient way to download a code file.",owner:"Golmote",require:"toolbar",noCSS:!0},"match-braces":{title:"Match braces",description:"Highlights matching braces.",owner:"RunDevelopment"},"diff-highlight":{title:"Diff Highlight",description:"Highlights the code inside diff blocks.",owner:"RunDevelopment",require:"diff"},"filter-highlight-all":{title:"Filter highlightAll",description:"Filters the elements the <code>highlightAll</code> and <code>highlightAllUnder</code> methods actually highlight.",owner:"RunDevelopment",noCSS:!0},treeview:{title:"Treeview",description:"A language with special styles to highlight file system tree structures.",owner:"Golmote"}}})},function(e,n,t){"use strict";var i=function(){var e=function(){};function n(e,n){Array.isArray(e)?e.forEach(n):null!=e&&n(e,0)}function t(e){for(var n={},t=0,i=e.length;t<i;t++)n[e[t]]=!0;return n}function i(e){var t={},i=[];function r(i,o){if(!(i in t)){o.push(i);var a=o.indexOf(i);if(a<o.length-1)throw new Error("Circular dependency: "+o.slice(a).join(" -> "));var l={},s=e[i];if(s){function c(n){if(!(n in e))throw new Error(i+" depends on an unknown component "+n);if(!(n in l))for(var a in r(n,o),l[n]=!0,t[n])l[a]=!0}n(s.require,c),n(s.optional,c),n(s.modify,c)}t[i]=l,o.pop()}}return function(e){var n=t[e];return n||(r(e,i),n=t[e]),n}}function r(e){for(var n in e)return!0;return!1}return function(o,a,l){var s=function(e){var n={};for(var t in e){var i=e[t];for(var r in i)if("meta"!=r){var o=i[r];n[r]="string"==typeof o?{title:o}:o}}return n}(o),c=function(e){var t;return function(i){if(i in e)return i;if(!t)for(var r in t={},e){var o=e[r];n(o&&o.alias,(function(n){if(n in t)throw new Error(n+" cannot be alias for both "+r+" and "+t[n]);if(n in e)throw new Error(n+" cannot be alias of "+r+" because it is a component.");t[n]=r}))}return t[i]||i}}(s);a=a.map(c),l=(l||[]).map(c);var u=t(a),p=t(l);a.forEach((function e(t){var i=s[t];n(i&&i.require,(function(n){n in p||(u[n]=!0,e(n))}))}));for(var d,f=i(s),m=u;r(m);){for(var g in d={},m){var h=s[g];n(h&&h.modify,(function(e){e in p&&(d[e]=!0)}))}for(var v in p)if(!(v in u))for(var b in f(v))if(b in u){d[v]=!0;break}for(var y in m=d)u[y]=!0}var k={getIds:function(){var e=[];return k.load((function(n){e.push(n)})),e},load:function(n,t){return function(n,t,i,r){var o=r?r.series:void 0,a=r?r.parallel:e,l={},s={};function c(e){if(e in l)return l[e];s[e]=!0;var r,u=[];for(var p in n(e))p in t&&u.push(p);if(0===u.length)r=i(e);else{var d=a(u.map((function(e){var n=c(e);return delete s[e],n})));o?r=o(d,(function(){return i(e)})):i(e)}return l[e]=r}for(var u in t)c(u);var p=[];for(var d in s)p.push(l[d]);return a(p)}(f,u,n,t)}};return k}}();e.exports=i},,,,function(e,n,t){"use strict";t.r(n);var i=t(0),r=t.n(i),o=t(30),a=t(10),l=t(20),s=t.n(l),c=function(e){var n=e.error,t=e.retry,i=e.pastDelay;return n?r.a.createElement("div",{style:{align:"center",color:"#fff",backgroundColor:"#fa383e",borderColor:"#fa383e",borderStyle:"solid",borderRadius:"0.25rem",borderWidth:"1px",boxSizing:"border-box",display:"block",padding:"1rem",flex:"0 0 50%",marginLeft:"25%",marginRight:"25%",marginTop:"5rem",maxWidth:"50%",width:"100%"}},r.a.createElement("p",null,n.message),r.a.createElement("div",null,r.a.createElement("button",{type:"button",onClick:t},"Retry"))):i?r.a.createElement("div",{style:{display:"flex",justifyContent:"center",alignItems:"center",height:"100vh"}},r.a.createElement("svg",{id:"loader",style:{width:128,height:110,position:"absolute",top:"calc(100vh - 64%)"},viewBox:"0 0 45 45",xmlns:"http://www.w3.org/2000/svg",stroke:"#61dafb"},r.a.createElement("g",{fill:"none",fillRule:"evenodd",transform:"translate(1 1)",strokeWidth:"2"},r.a.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.a.createElement("animate",{attributeName:"r",begin:"1.5s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-opacity",begin:"1.5s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-width",begin:"1.5s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.a.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.a.createElement("animate",{attributeName:"r",begin:"3s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-opacity",begin:"3s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.a.createElement("animate",{attributeName:"stroke-width",begin:"3s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.a.createElement("circle",{cx:"22",cy:"22",r:"8"},r.a.createElement("animate",{attributeName:"r",begin:"0s",dur:"1.5s",values:"6;1;2;3;4;5;6",calcMode:"linear",repeatCount:"indefinite"}))))):null},u=t(13),p={"00cb704f":[function(){return t.e(3).then(t.bind(null,65))},"@site/versioned_docs/version-2.0.0/deploy/docker_install_advanced.md",65],"012f72bc":[function(){return t.e(4).then(t.bind(null,66))},"@site/versioned_docs/version-1.2.11/pipeline/logstash.md",66],"0214960a":[function(){return t.e(5).then(t.bind(null,67))},"@site/versioned_docs/version-1.2.8/devel/docker.md",67],"02341ef2":[function(){return t.e(6).then(t.t.bind(null,68,3))},"~docs/default/version-1-2-11-metadata-prop-386.json",68],"0271802f":[function(){return t.e(7).then(t.bind(null,69))},"@site/versioned_docs/version-1.2.10/pipeline/logstash.md",69],"0327c8bf":[function(){return t.e(8).then(t.bind(null,70))},"@site/versioned_docs/version-1.2.11/deploy/bare_metal_install.md",70],"0381afdd":[function(){return t.e(9).then(t.bind(null,71))},"@site/versioned_docs/version-1.2.7/documentation_guide.md",71],"05c6fd0e":[function(){return t.e(10).then(t.bind(null,72))},"@site/versioned_docs/version-1.2.11/deploy/choosing.md",72],"066f8364":[function(){return t.e(11).then(t.bind(null,73))},"@site/versioned_docs/version-1.2.5/pipeline_importer.md",73],"07faddd5":[function(){return t.e(12).then(t.bind(null,74))},"@site/versioned_docs/version-1.2.7/pipeline/intro.md",74],"080a21b4":[function(){return t.e(13).then(t.bind(null,75))},"@site/docs/components/docker_first_steps.md",75],"0b8e461a":[function(){return t.e(14).then(t.bind(null,76))},"@site/versioned_docs/version-1.2.10/pipeline/tstat.md",76],"0d6d53ac":[function(){return t.e(15).then(t.bind(null,77))},"@site/versioned_docs/version-2.0.0/devel/documentation_guide.md",77],"16fcefd4":[function(){return t.e(16).then(t.bind(null,78))},"@site/docs/devel/tag.md",78],17896441:[function(){return Promise.all([t.e(0),t.e(1),t.e(17)]).then(t.bind(null,231))},"@theme/DocItem",231],"1bbe794e":[function(){return t.e(18).then(t.bind(null,83))},"@site/versioned_docs/version-2.0.0/deploy/docker_troubleshooting.md",83],"1be78505":[function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(161)]).then(t.bind(null,230))},"@theme/DocPage",230],"1cc43674":[function(){return t.e(19).then(t.bind(null,89))},"@site/docs/pipeline/nfdump.md",89],"1d038b8a":[function(){return t.e(20).then(t.bind(null,90))},"@site/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md",90],"1f3a3353":[function(){return t.e(21).then(t.bind(null,91))},"@site/docs/devel/docker.md",91],"21ee0bc4":[function(){return t.e(22).then(t.bind(null,92))},"@site/versioned_docs/version-1.2.6/pipeline_importer.md",92],"22b7278f":[function(){return t.e(23).then(t.bind(null,93))},"@site/docs/pipeline/intro.md",93],"22d86c11":[function(){return t.e(24).then(t.bind(null,94))},"@site/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md",94],"3179f6dc":[function(){return t.e(25).then(t.bind(null,95))},"@site/versioned_docs/version-1.2.5/deploy/choosing.md",95],"3309db24":[function(){return t.e(26).then(t.bind(null,96))},"@site/versioned_docs/version-1.2.7/pipeline/elastic_search.md",96],"33f6b1c9":[function(){return t.e(27).then(t.bind(null,97))},"@site/versioned_docs/version-1.2.8/pipeline/intro.md",97],"358e6759":[function(){return t.e(28).then(t.bind(null,98))},"@site/versioned_docs/version-1.2.7/components/docker_env.md",98],38962575:[function(){return t.e(29).then(t.bind(null,99))},"@site/versioned_docs/version-1.2.10/deploy/docker_upgrade.md",99],"38d1fe39":[function(){return t.e(30).then(t.bind(null,100))},"@site/versioned_docs/version-1.2.6/pipeline_logstash.md",100],"3914abfa":[function(){return t.e(31).then(t.bind(null,101))},"@site/versioned_docs/version-1.2.7/devel/pipeline_dataset.md",101],"40b960a4":[function(){return t.e(32).then(t.bind(null,102))},"@site/versioned_docs/version-1.2.9/devel/pipeline_dataset.md",102],"418618b7":[function(){return t.e(33).then(t.bind(null,103))},"@site/versioned_docs/version-2.0.0/deploy/docker_install_simple.md",103],"41980e0e":[function(){return t.e(34).then(t.bind(null,104))},"@site/versioned_docs/version-1.2.11/devel/docker.md",104],"42e094bc":[function(){return t.e(35).then(t.bind(null,105))},"@site/versioned_docs/version-1.2.8/deploy/docker_install_simple.md",105],"42f0e773":[function(){return t.e(36).then(t.bind(null,106))},"@site/versioned_docs/version-1.2.8/components/docker_pipeline.md",106],"42f7ba93":[function(){return t.e(37).then(t.bind(null,107))},"@site/versioned_docs/version-1.2.7/pipeline/nfdump.md",107],"43862f47":[function(){return t.e(38).then(t.bind(null,108))},"@site/versioned_docs/version-1.2.7/pipeline/importer.md",108],"46a600e4":[function(){return t.e(39).then(t.bind(null,109))},"@site/versioned_docs/version-1.2.10/components/docker_pipeline.md",109],"480c5911":[function(){return t.e(40).then(t.bind(null,110))},"@site/versioned_docs/version-1.2.5/DOCUMENTATION.md",110],49703371:[function(){return t.e(41).then(t.bind(null,111))},"@site/versioned_docs/version-1.2.8/deploy/bare_metal_install.md",111],"49e74cdd":[function(){return t.e(42).then(t.bind(null,112))},"@site/versioned_docs/version-1.2.10/pipeline/intro.md",112],"4a74d8e8":[function(){return t.e(43).then(t.bind(null,113))},"@site/versioned_docs/version-1.2.10/pipeline/elastic_search.md",113],"4ff24bf8":[function(){return t.e(44).then(t.bind(null,114))},"@site/versioned_docs/version-1.2.7/components/docker_upgrade.md",114],"4ffdf56a":[function(){return t.e(45).then(t.bind(null,115))},"@site/docs/devel/documentation_guide.md",115],50032990:[function(){return t.e(46).then(t.bind(null,116))},"@site/versioned_docs/version-1.2.9/pipeline/tstat.md",116],"514d766a":[function(){return t.e(47).then(t.bind(null,117))},"@site/versioned_docs/version-1.2.10/devel/documentation_guide.md",117],"53b0b5d8":[function(){return t.e(48).then(t.bind(null,118))},"@site/versioned_docs/version-2.0.0/devel/docker.md",118],"56776a50":[function(){return t.e(49).then(t.bind(null,119))},"@site/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md",119],"56a29d1a":[function(){return t.e(50).then(t.bind(null,120))},"@site/versioned_docs/version-1.2.9/deploy/bare_metal_install.md",120],"58d958f4":[function(){return t.e(51).then(t.bind(null,121))},"@site/versioned_docs/version-1.2.8/documentation_guide.md",121],"59305d05":[function(){return t.e(52).then(t.bind(null,122))},"@site/versioned_docs/version-2.0.0/pipeline/nfdump.md",122],"5a20f281":[function(){return t.e(53).then(t.bind(null,123))},"@site/versioned_docs/version-1.2.9/pipeline/elastic_search.md",123],"5b6c4cb7":[function(){return t.e(54).then(t.bind(null,124))},"@site/versioned_docs/version-1.2.11/pipeline/elastic_search.md",124],"5c4beb71":[function(){return t.e(55).then(t.bind(null,125))},"@site/versioned_docs/version-1.2.7/deploy/choosing.md",125],"5e9f5e1a":[function(){return Promise.resolve().then(t.bind(null,11))},"@generated/docusaurus.config",11],"5ee17acf":[function(){return t.e(56).then(t.bind(null,126))},"@site/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md",126],"6125e2e5":[function(){return t.e(57).then(t.bind(null,127))},"@site/versioned_docs/version-1.2.9/components/docker_env.md",127],"63f5fe43":[function(){return t.e(58).then(t.bind(null,128))},"@site/versioned_docs/version-1.2.10/devel/docker.md",128],"64269ed8":[function(){return t.e(59).then(t.bind(null,129))},"@site/docs/components/docker_env.md",129],"653602f1":[function(){return t.e(60).then(t.bind(null,130))},"@site/docs/deploy/docker_troubleshooting.md",130],"6546ef57":[function(){return t.e(61).then(t.bind(null,131))},"@site/versioned_docs/version-1.2.9/components/docker_upgrade.md",131],"66e31e6e":[function(){return t.e(62).then(t.bind(null,132))},"@site/docs/deploy/choosing.md",132],"69c1008e":[function(){return t.e(63).then(t.bind(null,133))},"@site/versioned_docs/version-1.2.11/components/docker_pipeline.md",133],"6a059f5b":[function(){return t.e(64).then(t.bind(null,134))},"@site/versioned_docs/version-1.2.8/pipeline/logstash.md",134],"6c279c12":[function(){return t.e(65).then(t.t.bind(null,135,3))},"~docs/default/version-1-2-5-metadata-prop-ebb.json",135],"6d913b5d":[function(){return t.e(66).then(t.bind(null,136))},"@site/versioned_docs/version-1.2.11/devel/tag.md",136],"734dc415":[function(){return t.e(67).then(t.bind(null,137))},"@site/docs/deploy/docker_install_simple.md",137],"799b1578":[function(){return t.e(68).then(t.bind(null,138))},"@site/versioned_docs/version-1.2.10/pipeline/importer.md",138],"7b85b03c":[function(){return t.e(69).then(t.bind(null,139))},"@site/versioned_docs/version-1.2.9/pipeline/nfdump.md",139],"7c78db09":[function(){return t.e(70).then(t.bind(null,140))},"@site/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md",140],"7d0b7755":[function(){return t.e(71).then(t.bind(null,141))},"@site/docs/pipeline/elastic_search.md",141],"7d380c4c":[function(){return t.e(72).then(t.t.bind(null,142,3))},"~docs/default/version-1-2-6-metadata-prop-dbf.json",142],"8084fbd4":[function(){return t.e(73).then(t.bind(null,143))},"@site/versioned_docs/version-2.0.0/pipeline/tstat.md",143],"832ca04e":[function(){return t.e(74).then(t.bind(null,144))},"@site/versioned_docs/version-1.2.8/pipeline/tstat.md",144],"83dbc9a5":[function(){return t.e(75).then(t.bind(null,145))},"@site/versioned_docs/version-1.2.5/devel/docker.md",145],"879923b2":[function(){return t.e(76).then(t.bind(null,146))},"@site/versioned_docs/version-1.2.7/deploy/docker_install_simple.md",146],"8997819f":[function(){return t.e(77).then(t.bind(null,147))},"@site/versioned_docs/version-1.2.11/components/docker_env.md",147],"89d5514b":[function(){return t.e(78).then(t.bind(null,148))},"@site/versioned_docs/version-2.0.0/components/docker_pipeline.md",148],"8bbfcfa9":[function(){return t.e(79).then(t.bind(null,149))},"@site/versioned_docs/version-1.2.10/deploy/choosing.md",149],"9001d127":[function(){return t.e(80).then(t.bind(null,150))},"@site/versioned_docs/version-2.0.0/pipeline/elastic_search.md",150],"906738b8":[function(){return t.e(81).then(t.t.bind(null,151,3))},"~docs/default/version-1-2-7-metadata-prop-988.json",151],"935f2afb":[function(){return t.e(82).then(t.t.bind(null,152,3))},"~docs/default/version-current-metadata-prop-751.json",152],96852331:[function(){return t.e(83).then(t.bind(null,153))},"@site/versioned_docs/version-1.2.11/pipeline/importer.md",153],"9720d299":[function(){return t.e(84).then(t.bind(null,154))},"@site/versioned_docs/version-1.2.9/pipeline/logstash.md",154],"97edd56e":[function(){return t.e(85).then(t.bind(null,155))},"@site/versioned_docs/version-1.2.7/devel/docker.md",155],"992819d4":[function(){return t.e(86).then(t.t.bind(null,156,3))},"~docs/default/version-1-2-8-metadata-prop-d68.json",156],"99d6017e":[function(){return t.e(87).then(t.bind(null,157))},"@site/versioned_docs/version-1.2.9/pipeline/importer.md",157],"9a5c9747":[function(){return t.e(88).then(t.bind(null,158))},"@site/versioned_docs/version-1.2.11/devel/pipeline_dataset.md",158],"9aa54785":[function(){return t.e(89).then(t.t.bind(null,159,3))},"~docs/default/version-2-0-0-metadata-prop-9ad.json",159],"9b505a1f":[function(){return t.e(90).then(t.bind(null,160))},"@site/versioned_docs/version-1.2.9/devel/docker.md",160],"9b6d3056":[function(){return t.e(91).then(t.bind(null,161))},"@site/versioned_docs/version-2.0.0/components/docker_env.md",161],"9bfdba8c":[function(){return t.e(92).then(t.bind(null,162))},"@site/versioned_docs/version-1.2.8/components/docker_first_steps.md",162],"9d5eac8e":[function(){return t.e(93).then(t.bind(null,163))},"@site/versioned_docs/version-1.2.11/devel/documentation_guide.md",163],"9e1ec80c":[function(){return t.e(94).then(t.bind(null,164))},"@site/versioned_docs/version-2.0.0/pipeline/logstash.md",164],"9fe55709":[function(){return t.e(95).then(t.bind(null,165))},"@site/versioned_docs/version-1.2.5/deploy/install.md",165],"9feaf8ab":[function(){return t.e(96).then(t.bind(null,166))},"@site/versioned_docs/version-1.2.6/deploy/docker_install.md",166],a04bc459:[function(){return t.e(97).then(t.bind(null,167))},"@site/versioned_docs/version-1.2.5/deploy/docker_install.md",167],a12215ea:[function(){return t.e(98).then(t.bind(null,168))},"@site/versioned_docs/version-1.2.11/pipeline/intro.md",168],a3c17fac:[function(){return t.e(99).then(t.bind(null,169))},"@site/versioned_docs/version-2.0.0/components/docker_first_steps.md",169],a55d7fc8:[function(){return t.e(100).then(t.bind(null,170))},"@site/versioned_docs/version-1.2.8/components/docker_upgrade.md",170],a7459da2:[function(){return t.e(101).then(t.bind(null,171))},"@site/versioned_docs/version-1.2.5/pipeline_logstash.md",171],aa98ec4a:[function(){return t.e(102).then(t.bind(null,172))},"@site/versioned_docs/version-1.2.6/devel/docker.md",172],aafd28b2:[function(){return t.e(103).then(t.bind(null,173))},"@site/versioned_docs/version-1.2.7/deploy/bare_metal_install.md",173],abbf3a21:[function(){return t.e(104).then(t.bind(null,174))},"@site/versioned_docs/version-1.2.8/components/docker_env.md",174],ac3b7d69:[function(){return t.e(105).then(t.bind(null,175))},"@site/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md",175],ad308aff:[function(){return t.e(106).then(t.bind(null,176))},"@site/versioned_docs/version-1.2.8/pipeline/importer.md",176],ae8a1364:[function(){return t.e(107).then(t.bind(null,177))},"@site/versioned_docs/version-1.2.10/components/docker_first_steps.md",177],af9cfdf8:[function(){return t.e(108).then(t.bind(null,178))},"@site/versioned_docs/version-2.0.0/pipeline/intro.md",178],b1bfe9f5:[function(){return t.e(109).then(t.bind(null,179))},"@site/versioned_docs/version-1.2.8/pipeline/nfdump.md",179],b3c114f4:[function(){return t.e(110).then(t.bind(null,180))},"@site/versioned_docs/version-1.2.9/components/docker_first_steps.md",180],b56f03c6:[function(){return t.e(111).then(t.bind(null,181))},"@site/versioned_docs/version-2.0.0/deploy/bare_metal_install.md",181],b83a1049:[function(){return t.e(112).then(t.bind(null,182))},"@site/versioned_docs/version-1.2.10/devel/tag.md",182],ba55e963:[function(){return t.e(113).then(t.bind(null,183))},"@site/docs/components/docker_pipeline.md",183],bb237b1e:[function(){return t.e(114).then(t.bind(null,184))},"@site/versioned_docs/version-1.2.7/pipeline/logstash.md",184],bd6c6f88:[function(){return t.e(115).then(t.bind(null,185))},"@site/docs/pipeline/logstash.md",185],bed7ab4e:[function(){return t.e(116).then(t.bind(null,186))},"@site/versioned_docs/version-1.2.8/pipeline/elastic_search.md",186],c0bd86e8:[function(){return t.e(117).then(t.bind(null,187))},"@site/versioned_docs/version-1.2.5/pipeline.md",187],c3f49368:[function(){return t.e(118).then(t.bind(null,188))},"@site/versioned_docs/version-1.2.9/pipeline/intro.md",188],c4f5d8e4:[function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(119)]).then(t.bind(null,189))},"@site/src/pages/index.js",189],c51f4c72:[function(){return t.e(120).then(t.bind(null,191))},"@site/versioned_docs/version-1.2.9/deploy/docker_install_simple.md",191],cbd1c0d0:[function(){return t.e(121).then(t.bind(null,192))},"@site/docs/deploy/docker_install_advanced.md",192],cd902920:[function(){return t.e(122).then(t.bind(null,193))},"@site/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md",193],ceb45c47:[function(){return t.e(123).then(t.bind(null,194))},"@site/docs/deploy/docker_upgrade.md",194],d1597237:[function(){return t.e(124).then(t.bind(null,195))},"@site/versioned_docs/version-1.2.9/components/docker_pipeline.md",195],d1b2eded:[function(){return t.e(125).then(t.bind(null,196))},"@site/versioned_docs/version-2.0.0/pipeline/importer.md",196],d2c255b4:[function(){return t.e(126).then(t.bind(null,197))},"@site/versioned_docs/version-2.0.0/deploy/choosing.md",197],d2c4f39e:[function(){return t.e(127).then(t.bind(null,198))},"@site/versioned_docs/version-1.2.8/deploy/choosing.md",198],d3556f45:[function(){return t.e(128).then(t.bind(null,199))},"@site/docs/pipeline/importer.md",199],d48b255e:[function(){return t.e(129).then(t.bind(null,200))},"@site/versioned_docs/version-1.2.11/deploy/docker_upgrade.md",200],d5087a67:[function(){return t.e(130).then(t.t.bind(null,201,3))},"~docs/default/version-1-2-10-metadata-prop-e71.json",201],d5140579:[function(){return t.e(131).then(t.bind(null,202))},"@site/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md",202],d5479f42:[function(){return t.e(132).then(t.bind(null,203))},"@site/versioned_docs/version-1.2.6/deploy/choosing.md",203],d71ff009:[function(){return t.e(133).then(t.bind(null,204))},"@site/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md",204],d75898ab:[function(){return t.e(134).then(t.t.bind(null,205,3))},"~docs/default/version-1-2-9-metadata-prop-33d.json",205],d91f9def:[function(){return t.e(135).then(t.bind(null,206))},"@site/versioned_docs/version-1.2.6/DOCUMENTATION.md",206],db0ed613:[function(){return t.e(136).then(t.bind(null,207))},"@site/versioned_docs/version-1.2.9/documentation_guide.md",207],dd1a9f26:[function(){return t.e(137).then(t.bind(null,208))},"@site/versioned_docs/version-1.2.11/deploy/docker_install_simple.md",208],ded423ee:[function(){return t.e(138).then(t.bind(null,209))},"@site/versioned_docs/version-2.0.0/devel/pipeline_dataset.md",209],e00e17a0:[function(){return t.e(139).then(t.bind(null,210))},"@site/versioned_docs/version-1.2.8/devel/pipeline_dataset.md",210],e10970d8:[function(){return t.e(140).then(t.bind(null,211))},"@site/versioned_docs/version-1.2.10/components/docker_env.md",211],e186625e:[function(){return t.e(141).then(t.bind(null,212))},"@site/versioned_docs/version-1.2.7/pipeline/tstat.md",212],e2be698d:[function(){return t.e(142).then(t.bind(null,213))},"@site/versioned_docs/version-1.2.6/deploy/install.md",213],e2c161d9:[function(){return t.e(143).then(t.bind(null,214))},"@site/docs/pipeline/tstat.md",214],e30bbbcf:[function(){return t.e(144).then(t.bind(null,215))},"@site/versioned_docs/version-1.2.11/pipeline/nfdump.md",215],e3649aa7:[function(){return t.e(145).then(t.bind(null,216))},"@site/versioned_docs/version-2.0.0/deploy/docker_upgrade.md",216],e436ce66:[function(){return t.e(146).then(t.bind(null,217))},"@site/versioned_docs/version-1.2.7/components/docker_pipeline.md",217],e4c61278:[function(){return t.e(147).then(t.bind(null,218))},"@site/versioned_docs/version-2.0.0/devel/tag.md",218],e5ce8f83:[function(){return t.e(148).then(t.bind(null,219))},"@site/docs/deploy/bare_metal_install.md",219],e746408e:[function(){return t.e(149).then(t.bind(null,220))},"@site/versioned_docs/version-1.2.10/deploy/docker_install_simple.md",220],e7670743:[function(){return t.e(150).then(t.bind(null,221))},"@site/versioned_docs/version-1.2.10/deploy/bare_metal_install.md",221],e987ec87:[function(){return t.e(151).then(t.bind(null,222))},"@site/versioned_docs/version-1.2.11/components/docker_first_steps.md",222],ebfaece6:[function(){return t.e(152).then(t.bind(null,223))},"@site/versioned_docs/version-1.2.6/pipeline.md",223],f1ddb0ed:[function(){return t.e(153).then(t.bind(null,224))},"@site/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md",224],f485eca7:[function(){return t.e(154).then(t.bind(null,225))},"@site/versioned_docs/version-1.2.10/devel/pipeline_dataset.md",225],f52ba0ac:[function(){return t.e(155).then(t.bind(null,226))},"@site/versioned_docs/version-1.2.11/pipeline/tstat.md",226],f64b3473:[function(){return t.e(156).then(t.bind(null,227))},"@site/versioned_docs/version-1.2.9/deploy/choosing.md",227],f8bc0717:[function(){return t.e(157).then(t.bind(null,228))},"@site/versioned_docs/version-1.2.10/pipeline/nfdump.md",228],fa16f0fc:[function(){return t.e(158).then(t.bind(null,229))},"@site/docs/devel/pipeline_dataset.md",229]};var d=function(e){var n={};return function e(t,i){Object.keys(t).forEach((function(r){var o=t[r],a=i?i+"."+r:r;"object"===typeof o&&!!o&&Object.keys(o).length?e(o,a):n[a]=o}))}(e),n};var f=function(e,n){if("*"===e)return s()({loading:c,loader:function(){return Promise.all([t.e(0),t.e(1),t.e(2),t.e(162)]).then(t.bind(null,243))}});var i=u[e+"-"+n],o=[],a=[],l={},f=d(i);return Object.keys(f).forEach((function(e){var n=p[f[e]];n&&(l[e]=n[0],o.push(n[1]),a.push(n[2]))})),s.a.Map({loading:c,loader:l,modules:o,webpack:function(){return a},render:function(e,n){var t=JSON.parse(JSON.stringify(i));Object.keys(e).forEach((function(n){for(var i=t,r=n.split("."),o=0;o<r.length-1;o+=1)i=i[r[o]];i[r[r.length-1]]=e[n].default;var a=Object.keys(e[n]).filter((function(e){return"default"!==e}));a&&a.length&&a.forEach((function(t){i[r[r.length-1]][t]=e[n][t]}))}));var o=t.component;return delete t.component,r.a.createElement(o,Object.assign({},t,n))}})},m=[{path:"/netsage-pipeline/",component:f("/netsage-pipeline/","421"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10",component:f("/netsage-pipeline/docs/1.2.10","296"),routes:[{path:"/netsage-pipeline/docs/1.2.10/components/docker_env",component:f("/netsage-pipeline/docs/1.2.10/components/docker_env","63f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.10/components/docker_first_steps","059"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.10/components/docker_pipeline","a0b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install","ec8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.10/deploy/choose_install","fb2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced","a83"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple","15f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot","e2e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade","b01"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.10/devel/dev_dataset","7fa"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide","f40"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag","b8f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/devel/docusaurus",component:f("/netsage-pipeline/docs/1.2.10/devel/docusaurus","bcf"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.10/pipeline/elastic","c07"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.10/pipeline/importer","3c0"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.10/pipeline/intro","3b3"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.10/pipeline/logstash","116"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.10/pipeline/nfdump","f4d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.10/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.10/pipeline/tstat","4c2"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.11",component:f("/netsage-pipeline/docs/1.2.11","d04"),routes:[{path:"/netsage-pipeline/docs/1.2.11/components/docker_env",component:f("/netsage-pipeline/docs/1.2.11/components/docker_env","1d6"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.11/components/docker_first_steps","62e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.11/components/docker_pipeline","6b1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install","c46"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.11/deploy/choose_install","f72"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced","0a8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple","c8a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot","a22"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade","9ae"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.11/devel/dev_dataset","ba4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide","80d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag","681"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/devel/docusaurus",component:f("/netsage-pipeline/docs/1.2.11/devel/docusaurus","e33"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.11/pipeline/elastic","ef5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.11/pipeline/importer","5cc"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.11/pipeline/intro","5fd"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.11/pipeline/logstash","0d3"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.11/pipeline/nfdump","9b4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.11/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.11/pipeline/tstat","3cb"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.5",component:f("/netsage-pipeline/docs/1.2.5","c52"),routes:[{path:"/netsage-pipeline/docs/1.2.5/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.5/deploy/choose_install","3ce"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/deploy/docker_install",component:f("/netsage-pipeline/docs/1.2.5/deploy/docker_install","d98"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/deploy/install",component:f("/netsage-pipeline/docs/1.2.5/deploy/install","47c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/devel/docker",component:f("/netsage-pipeline/docs/1.2.5/devel/docker","0e4"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/docusaurus",component:f("/netsage-pipeline/docs/1.2.5/docusaurus","882"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline",component:f("/netsage-pipeline/docs/1.2.5/pipeline","8ff"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline_importer",component:f("/netsage-pipeline/docs/1.2.5/pipeline_importer","93b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.5/pipeline_logstash",component:f("/netsage-pipeline/docs/1.2.5/pipeline_logstash","9ed"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.6",component:f("/netsage-pipeline/docs/1.2.6","489"),routes:[{path:"/netsage-pipeline/docs/1.2.6/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.6/deploy/choose_install","0d1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/deploy/docker_install",component:f("/netsage-pipeline/docs/1.2.6/deploy/docker_install","4df"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/deploy/install",component:f("/netsage-pipeline/docs/1.2.6/deploy/install","8ee"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/devel/docker",component:f("/netsage-pipeline/docs/1.2.6/devel/docker","d95"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/docusaurus",component:f("/netsage-pipeline/docs/1.2.6/docusaurus","312"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline",component:f("/netsage-pipeline/docs/1.2.6/pipeline","7ea"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline_importer",component:f("/netsage-pipeline/docs/1.2.6/pipeline_importer","c76"),exact:!0},{path:"/netsage-pipeline/docs/1.2.6/pipeline_logstash",component:f("/netsage-pipeline/docs/1.2.6/pipeline_logstash","ee6"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.7",component:f("/netsage-pipeline/docs/1.2.7","3e0"),routes:[{path:"/netsage-pipeline/docs/1.2.7/components/docker_env",component:f("/netsage-pipeline/docs/1.2.7/components/docker_env","fb9"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.7/components/docker_pipeline","28e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.7/components/docker_upgrade","aca"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install","88b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.7/deploy/choose_install","9c2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced","a68"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple","c15"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot","f5e"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.7/devel/dev_dataset","c2f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide","13c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/docusaurus",component:f("/netsage-pipeline/docs/1.2.7/docusaurus","fba"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.7/pipeline/elastic","545"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.7/pipeline/importer","457"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.7/pipeline/intro","d5a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.7/pipeline/logstash","144"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.7/pipeline/nfdump","6d8"),exact:!0},{path:"/netsage-pipeline/docs/1.2.7/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.7/pipeline/tstat","d6c"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.8",component:f("/netsage-pipeline/docs/1.2.8","51d"),routes:[{path:"/netsage-pipeline/docs/1.2.8/components/docker_env",component:f("/netsage-pipeline/docs/1.2.8/components/docker_env","f3f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.8/components/docker_first_steps","d25"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.8/components/docker_pipeline","7d7"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.8/components/docker_upgrade","db5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install","a0d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.8/deploy/choose_install","298"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced","150"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple","f57"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot","a18"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.8/devel/dev_dataset","744"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide","e12"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/docusaurus",component:f("/netsage-pipeline/docs/1.2.8/docusaurus","57f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.8/pipeline/elastic","e1a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.8/pipeline/importer","b38"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.8/pipeline/intro","c0b"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.8/pipeline/logstash","00d"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.8/pipeline/nfdump","6f1"),exact:!0},{path:"/netsage-pipeline/docs/1.2.8/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.8/pipeline/tstat","59a"),exact:!0}]},{path:"/netsage-pipeline/docs/1.2.9",component:f("/netsage-pipeline/docs/1.2.9","cb5"),routes:[{path:"/netsage-pipeline/docs/1.2.9/components/docker_env",component:f("/netsage-pipeline/docs/1.2.9/components/docker_env","e31"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_first_steps",component:f("/netsage-pipeline/docs/1.2.9/components/docker_first_steps","253"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_pipeline",component:f("/netsage-pipeline/docs/1.2.9/components/docker_pipeline","6e2"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/components/docker_upgrade",component:f("/netsage-pipeline/docs/1.2.9/components/docker_upgrade","8be"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install","31c"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/choose_install",component:f("/netsage-pipeline/docs/1.2.9/deploy/choose_install","5ff"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced","711"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple","536"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot","7d0"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/devel/dev_dataset",component:f("/netsage-pipeline/docs/1.2.9/devel/dev_dataset","e4a"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide","b39"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/docusaurus",component:f("/netsage-pipeline/docs/1.2.9/docusaurus","bed"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/elastic",component:f("/netsage-pipeline/docs/1.2.9/pipeline/elastic","bd5"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/importer",component:f("/netsage-pipeline/docs/1.2.9/pipeline/importer","90f"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/intro",component:f("/netsage-pipeline/docs/1.2.9/pipeline/intro","e91"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/logstash",component:f("/netsage-pipeline/docs/1.2.9/pipeline/logstash","9e9"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/nfdump",component:f("/netsage-pipeline/docs/1.2.9/pipeline/nfdump","220"),exact:!0},{path:"/netsage-pipeline/docs/1.2.9/pipeline/tstat",component:f("/netsage-pipeline/docs/1.2.9/pipeline/tstat","341"),exact:!0}]},{path:"/netsage-pipeline/docs/next",component:f("/netsage-pipeline/docs/next","8b0"),routes:[{path:"/netsage-pipeline/docs/next/components/docker_env",component:f("/netsage-pipeline/docs/next/components/docker_env","cfe"),exact:!0},{path:"/netsage-pipeline/docs/next/components/docker_first_steps",component:f("/netsage-pipeline/docs/next/components/docker_first_steps","e72"),exact:!0},{path:"/netsage-pipeline/docs/next/components/docker_pipeline",component:f("/netsage-pipeline/docs/next/components/docker_pipeline","5eb"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/next/deploy/bare_metal_install","141"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/choose_install",component:f("/netsage-pipeline/docs/next/deploy/choose_install","d51"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/next/deploy/docker_install_advanced","de0"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/next/deploy/docker_install_simple","72d"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/next/deploy/docker_troubleshoot","593"),exact:!0},{path:"/netsage-pipeline/docs/next/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/next/deploy/docker_upgrade","68f"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/dev_dataset",component:f("/netsage-pipeline/docs/next/devel/dev_dataset","091"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/next/devel/docker_dev_guide","85f"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/next/devel/docker_dev_tag","e93"),exact:!0},{path:"/netsage-pipeline/docs/next/devel/docusaurus",component:f("/netsage-pipeline/docs/next/devel/docusaurus","702"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/elastic",component:f("/netsage-pipeline/docs/next/pipeline/elastic","533"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/importer",component:f("/netsage-pipeline/docs/next/pipeline/importer","063"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/intro",component:f("/netsage-pipeline/docs/next/pipeline/intro","f04"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/logstash",component:f("/netsage-pipeline/docs/next/pipeline/logstash","aec"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/nfdump",component:f("/netsage-pipeline/docs/next/pipeline/nfdump","543"),exact:!0},{path:"/netsage-pipeline/docs/next/pipeline/tstat",component:f("/netsage-pipeline/docs/next/pipeline/tstat","474"),exact:!0}]},{path:"/netsage-pipeline/docs",component:f("/netsage-pipeline/docs","89b"),routes:[{path:"/netsage-pipeline/docs/components/docker_env",component:f("/netsage-pipeline/docs/components/docker_env","74f"),exact:!0},{path:"/netsage-pipeline/docs/components/docker_first_steps",component:f("/netsage-pipeline/docs/components/docker_first_steps","c20"),exact:!0},{path:"/netsage-pipeline/docs/components/docker_pipeline",component:f("/netsage-pipeline/docs/components/docker_pipeline","2f6"),exact:!0},{path:"/netsage-pipeline/docs/deploy/bare_metal_install",component:f("/netsage-pipeline/docs/deploy/bare_metal_install","faa"),exact:!0},{path:"/netsage-pipeline/docs/deploy/choose_install",component:f("/netsage-pipeline/docs/deploy/choose_install","e79"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_install_advanced",component:f("/netsage-pipeline/docs/deploy/docker_install_advanced","161"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_install_simple",component:f("/netsage-pipeline/docs/deploy/docker_install_simple","64b"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_troubleshoot",component:f("/netsage-pipeline/docs/deploy/docker_troubleshoot","bfd"),exact:!0},{path:"/netsage-pipeline/docs/deploy/docker_upgrade",component:f("/netsage-pipeline/docs/deploy/docker_upgrade","732"),exact:!0},{path:"/netsage-pipeline/docs/devel/dev_dataset",component:f("/netsage-pipeline/docs/devel/dev_dataset","4f7"),exact:!0},{path:"/netsage-pipeline/docs/devel/docker_dev_guide",component:f("/netsage-pipeline/docs/devel/docker_dev_guide","e38"),exact:!0},{path:"/netsage-pipeline/docs/devel/docker_dev_tag",component:f("/netsage-pipeline/docs/devel/docker_dev_tag","194"),exact:!0},{path:"/netsage-pipeline/docs/devel/docusaurus",component:f("/netsage-pipeline/docs/devel/docusaurus","039"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/elastic",component:f("/netsage-pipeline/docs/pipeline/elastic","023"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/importer",component:f("/netsage-pipeline/docs/pipeline/importer","91a"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/intro",component:f("/netsage-pipeline/docs/pipeline/intro","c00"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/logstash",component:f("/netsage-pipeline/docs/pipeline/logstash","105"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/nfdump",component:f("/netsage-pipeline/docs/pipeline/nfdump","d28"),exact:!0},{path:"/netsage-pipeline/docs/pipeline/tstat",component:f("/netsage-pipeline/docs/pipeline/tstat","c1d"),exact:!0}]},{path:"*",component:f("*")}],g=t(8),h=t(11),v=t(31),b=t(32),y=t(26),k=t(33),w=t(27),_=t(14),x=t(4),E=t(2),S=t(15),T=t.n(S),P=[t(18),t(18),t(18),t(47),t(55),t(51)];function C(e){for(var n=arguments.length,t=new Array(n>1?n-1:0),i=1;i<n;i++)t[i-1]=arguments[i];P.forEach((function(n){var i,r,o=null!==(r=null===(i=null==n?void 0:n.default)||void 0===i?void 0:i[e])&&void 0!==r?r:n[e];o&&o.apply(void 0,t)}))}var O={onRouteUpdate:function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];C.apply(void 0,["onRouteUpdate"].concat(n))},onRouteUpdateDelayed:function(){for(var e=arguments.length,n=new Array(e),t=0;t<e;t++)n[t]=arguments[t];C.apply(void 0,["onRouteUpdateDelayed"].concat(n))}},A=t(9);function R(e,n){var t=Object(A.a)(e,n);return Promise.all(t.map((function(e){var n=e.route.component;if(n&&n.preload)return n.preload()})))}var N={};var I=function(e){if(N[e.pathname])return Object.assign(Object.assign({},e),{pathname:N[e.pathname]});var n=e.pathname||"/";return""===(n=n.trim().replace(/\/index\.html$/,""))&&(n="/"),N[e.pathname]=n,Object.assign(Object.assign({},e),{pathname:n})};t(52);T.a.configure({showSpinner:!1});var L=function(e){function n(n){var t;return(t=e.call(this,n)||this).previousLocation=null,t.progressBarTimeout=null,t.state={nextRouteHasLoaded:!0},t}Object(x.a)(n,e);var t=n.prototype;return t.shouldComponentUpdate=function(e,n){var t=this,i=e.location!==this.props.location,r=this.props,o=r.routes,a=r.delay,l=void 0===a?1e3:a;if(i){var s=I(e.location);return this.startProgressBar(l),this.previousLocation=I(this.props.location),this.setState({nextRouteHasLoaded:!1}),R(o,s.pathname).then((function(){O.onRouteUpdate({previousLocation:t.previousLocation,location:s}),t.previousLocation=null,t.setState({nextRouteHasLoaded:!0},t.stopProgressBar);var e=s.hash;if(e){var n=decodeURIComponent(e.substring(1)),i=document.getElementById(n);i&&i.scrollIntoView()}else window.scrollTo(0,0)})).catch((function(e){return console.warn(e)})),!1}return!!n.nextRouteHasLoaded},t.clearProgressBarTimeout=function(){this.progressBarTimeout&&(clearTimeout(this.progressBarTimeout),this.progressBarTimeout=null)},t.startProgressBar=function(e){var n=this;this.clearProgressBarTimeout(),this.progressBarTimeout=setTimeout((function(){O.onRouteUpdateDelayed({location:I(n.props.location)}),T.a.start()}),e)},t.stopProgressBar=function(){this.clearProgressBarTimeout(),T.a.done()},t.render=function(){var e=this.props,n=e.children,t=e.location;return r.a.createElement(E.d,{location:I(t),render:function(){return n}})},n}(r.a.Component),D=Object(E.o)(L),j=t(23),M=t(25),F=t(16),B=(t(53),"docusaurus-base-url-issue-banner-container"),z="docusaurus-base-url-issue-banner-suggestion-container",U="__DOCUSAURUS_INSERT_BASEURL_BANNER";function $(e){return"\nwindow['"+U+"'] = true;\n\ndocument.addEventListener('DOMContentLoaded', maybeInsertBanner);\n\nfunction maybeInsertBanner() {\n  var shouldInsert = window['"+U+"'];\n  shouldInsert && insertBanner();\n}\n\nfunction insertBanner() {\n  var bannerContainer = document.getElementById('"+B+"');\n  if (!bannerContainer) {\n    return;\n  }\n  var bannerHtml = "+JSON.stringify(function(e){return'\n<div id="docusaurus-base-url-issue-banner" style="border: thick solid red; background-color: rgb(255, 230, 179); margin: 20px; padding: 20px; font-size: 20px;">\n   <p style="font-weight: bold; font-size: 30px;">Your Docusaurus site did not load properly.</p>\n   <p>A very common reason is a wrong site <a href="https://v2.docusaurus.io/docs/docusaurus.config.js/#baseurl" style="font-weight: bold;">baseUrl configuration</a>.</p>\n   <p>Current configured baseUrl = <span style="font-weight: bold; color: red;">'+e+"</span> "+("/"===e?" (default value)":"")+'</p>\n   <p>We suggest trying baseUrl = <span id="'+z+'" style="font-weight: bold; color: green;"></span></p>\n</div>\n'}(e)).replace(/</g,"\\<")+";\n  bannerContainer.innerHTML = bannerHtml;\n  var suggestionContainer = document.getElementById('"+z+"');\n  var actualHomePagePath = window.location.pathname;\n  var suggestedBaseUrl = actualHomePagePath.substr(-1) === '/'\n        ? actualHomePagePath\n        : actualHomePagePath + '/';\n  suggestionContainer.innerHTML = suggestedBaseUrl;\n}\n"}function G(){var e=Object(F.default)().siteConfig.baseUrl;return Object(i.useLayoutEffect)((function(){window[U]=!1}),[]),r.a.createElement(r.a.Fragment,null,!g.a.canUseDOM&&r.a.createElement(M.a,null,r.a.createElement("script",null,$(e))),r.a.createElement("div",{id:B}))}function q(){var e=Object(F.default)().siteConfig,n=e.baseUrl,t=e.baseUrlIssueBanner,i=Object(j.useLocation)().pathname;return t&&i===n?r.a.createElement(G,null):null}var H=function(e){var n=e.children;return r.a.createElement(r.a.Fragment,null,n)};var W=function(){var e=Object(i.useState)(!1),n=e[0],t=e[1];return Object(i.useEffect)((function(){t(!0)}),[]),r.a.createElement(_.a.Provider,{value:{siteConfig:h.default,siteMetadata:k,globalData:v,i18n:b,codeTranslations:y,isClient:n}},r.a.createElement(H,null,r.a.createElement(q,null),r.a.createElement(D,{routes:m},Object(w.a)(m))))};var V=function(e){if("undefined"==typeof document)return!1;var n=document.createElement("link");try{if(n.relList&&"function"==typeof n.relList.supports)return n.relList.supports(e)}catch(t){return!1}return!1}("prefetch")?function(e){return new Promise((function(n,t){if("undefined"!=typeof document){var i=document.createElement("link");i.setAttribute("rel","prefetch"),i.setAttribute("href",e),i.onload=n,i.onerror=t,(document.getElementsByTagName("head")[0]||document.getElementsByName("script")[0].parentNode).appendChild(i)}else t()}))}:function(e){return new Promise((function(n,t){var i=new XMLHttpRequest;i.open("GET",e,!0),i.withCredentials=!0,i.onload=function(){200===i.status?n():t()},i.send(null)}))},K={};var Y=function(e){return new Promise((function(n){K[e]?n():V(e).then((function(){n(),K[e]=!0})).catch((function(){}))}))},Q={},X={},J=function(){return!(!("connection"in navigator)||-1===(navigator.connection.effectiveType||"").indexOf("2g")||!navigator.connection.saveData)},Z=function(e){return Array.prototype.concat.apply([],e)},ee={prefetch:function(e){if(!function(e){return!J()&&!X[e]&&!Q[e]}(e))return!1;Q[e]=!0;var n=Object(A.a)(m,e);return Z(n.map((function(e){return n=e.route.path,Z(Object.entries(u).filter((function(e){return e[0].replace(/(-[^-]+)$/,"")===n})).map((function(e){var n=e[1];return Object.values(d(n))})));var n}))).forEach((function(e){var n=t.gca(e);n&&!/undefined/.test(n)&&Y(n)})),!0},preload:function(e){return!!function(e){return!J()&&!X[e]}(e)&&(X[e]=!0,R(m,e),!0)}};if(g.a.canUseDOM){window.docusaurus=ee;var ne=o.hydrate;R(m,window.location.pathname).then((function(){ne(r.a.createElement(a.a,null,r.a.createElement(W,null)),document.getElementById("__docusaurus"))}))}},function(e,n,t){"use strict";t.r(n);var i=t(24),r=t(8),o=t(11);(function(e){if(r.a.canUseDOM){var n=o.default.themeConfig.prism,i=(n=void 0===n?{}:n).additionalLanguages,a=void 0===i?[]:i;window.Prism=e,a.forEach((function(e){t(19)("./prism-"+e)})),delete window.Prism}})(i.a)}],[[38,160,0]]]);
\ No newline at end of file
diff --git a/assets/js/runtime~main.09475183.js b/assets/js/runtime~main.09475183.js
new file mode 100644
index 00000000..3e2c4ea2
--- /dev/null
+++ b/assets/js/runtime~main.09475183.js
@@ -0,0 +1 @@
+!function(e){function f(f){for(var d,r,t=f[0],n=f[1],o=f[2],u=0,l=[];u<t.length;u++)r=t[u],Object.prototype.hasOwnProperty.call(c,r)&&c[r]&&l.push(c[r][0]),c[r]=0;for(d in n)Object.prototype.hasOwnProperty.call(n,d)&&(e[d]=n[d]);for(i&&i(f);l.length;)l.shift()();return b.push.apply(b,o||[]),a()}function a(){for(var e,f=0;f<b.length;f++){for(var a=b[f],d=!0,r=1;r<a.length;r++){var n=a[r];0!==c[n]&&(d=!1)}d&&(b.splice(f--,1),e=t(t.s=a[0]))}return e}var d={},c={156:0},b=[];function r(e){return t.p+"assets/js/"+({3:"012f72bc",4:"0214960a",5:"02341ef2",6:"0271802f",7:"0327c8bf",8:"0381afdd",9:"05c6fd0e",10:"066f8364",11:"07faddd5",12:"080a21b4",13:"092c2d1d",14:"0b8e461a",15:"0dd3dba6",16:"12ba72b5",17:"12ecfc4a",18:"16fcefd4",19:"17896441",20:"1cc43674",21:"1d038b8a",22:"1f3a3353",23:"21ee0bc4",24:"22b7278f",25:"22d86c11",26:"3179f6dc",27:"3309db24",28:"33f6b1c9",29:"358e6759",30:"38962575",31:"38d1fe39",32:"3914abfa",33:"40b960a4",34:"41980e0e",35:"42e094bc",36:"42f0e773",37:"42f7ba93",38:"43862f47",39:"46a600e4",40:"480c5911",41:"49703371",42:"49e74cdd",43:"4a74d8e8",44:"4ff24bf8",45:"4ffdf56a",46:"50032990",47:"514d766a",48:"56776a50",49:"56a29d1a",50:"58d958f4",51:"5a20f281",52:"5b6c4cb7",53:"5c4beb71",54:"5ee17acf",55:"5ef24f1d",56:"6125e2e5",57:"63503174",58:"63f5fe43",59:"64269ed8",60:"653602f1",61:"6546ef57",62:"69c1008e",63:"6a059f5b",64:"6c279c12",65:"6d913b5d",66:"6e2a1c72",67:"6e3166bc",68:"71e5fd50",69:"734dc415",70:"799b1578",71:"7b85b03c",72:"7c78db09",73:"7d0b7755",74:"7d380c4c",75:"832ca04e",76:"83dbc9a5",77:"86d09e6e",78:"879923b2",79:"8997819f",80:"8b301107",81:"8bbfcfa9",82:"906738b8",83:"9191c864",84:"935f2afb",85:"96852331",86:"9720d299",87:"97edd56e",88:"992819d4",89:"99d6017e",90:"9a5c9747",91:"9b505a1f",92:"9bfdba8c",93:"9d5eac8e",94:"9f628d52",95:"9fe55709",96:"9feaf8ab",97:"a04bc459",98:"a12215ea",99:"a55d7fc8",100:"a7459da2",101:"aa98ec4a",102:"aafd28b2",103:"abbf3a21",104:"ac3b7d69",105:"ad308aff",106:"ae8a1364",107:"b1bfe9f5",108:"b3c114f4",109:"b83a1049",110:"ba55e963",111:"bb237b1e",112:"bd6c6f88",113:"bed7ab4e",114:"c0bd86e8",115:"c3f49368",116:"c4f5d8e4",117:"c51f4c72",118:"cbd1c0d0",119:"cd902920",120:"ceb45c47",121:"d1597237",122:"d2c4f39e",123:"d3556f45",124:"d48b255e",125:"d5087a67",126:"d5140579",127:"d5479f42",128:"d71ff009",129:"d74c151a",130:"d75898ab",131:"d91f9def",132:"db0ed613",133:"dd1a9f26",134:"e00e17a0",135:"e10970d8",136:"e186625e",137:"e2be698d",138:"e2c161d9",139:"e30bbbcf",140:"e436ce66",141:"e746408e",142:"e7670743",143:"e987ec87",144:"ebfaece6",145:"f0b0282f",146:"f1ddb0ed",147:"f2cce206",148:"f485eca7",149:"f52ba0ac",150:"f64b3473",151:"f8bc0717",152:"f9995f72",153:"fa16f0fc",154:"ff1d9e71"}[e]||e)+"."+{1:"03db48d6",2:"f7e8da3d",3:"36b7fe96",4:"7e6a0dc4",5:"76f0a379",6:"e2ca0056",7:"4ed718d8",8:"de4385a8",9:"27777f60",10:"bfb615a4",11:"b2854b31",12:"616746c2",13:"97793caf",14:"4a1a6155",15:"3dbbc1f3",16:"4b8388c3",17:"a1b9aa60",18:"9eb05a6f",19:"104e0abb",20:"f9080c78",21:"936f367a",22:"117952bc",23:"03721b9e",24:"58242775",25:"2a981864",26:"bedc2974",27:"f6153687",28:"798766ac",29:"19972cb0",30:"8edb151f",31:"e28e353a",32:"df468ae2",33:"422649da",34:"d717c996",35:"04cc4a66",36:"73889b7e",37:"e851b609",38:"7a6ac57b",39:"790fa3f6",40:"48d5baca",41:"0286d91d",42:"00e19770",43:"d930cabd",44:"c0099967",45:"f556f662",46:"47d0b7f0",47:"ca35f79e",48:"009ef339",49:"de0b8997",50:"572280db",51:"84ff2815",52:"1e5700c2",53:"b666c6b3",54:"4e665490",55:"6c97f268",56:"372c6163",57:"24f9a6f6",58:"faa7d00d",59:"18f9818d",60:"d686a47d",61:"b480f0cc",62:"6660742f",63:"57b36ba4",64:"225c0b15",65:"bb807553",66:"7c78a48b",67:"fa40017d",68:"9e08f691",69:"9fadde42",70:"ad714f78",71:"6e3a64bd",72:"5ea6e962",73:"c19774b1",74:"b92baa88",75:"24044575",76:"4a4e5652",77:"43e743b3",78:"90f3d9b4",79:"aa7254b4",80:"d02fcb00",81:"dc015f8e",82:"3a283baa",83:"9b61c372",84:"9f248d32",85:"822f904f",86:"9abe8149",87:"d5ab66b5",88:"2993e926",89:"cb54715a",90:"85ad9bca",91:"f4b85bdc",92:"b2391bbd",93:"6a3edbf9",94:"08c37d03",95:"e9c7bfda",96:"8c912915",97:"dcff4f83",98:"1ea3e4ef",99:"d2b40c2a",100:"2d033572",101:"39b5137c",102:"fb1f547f",103:"2995f8fb",104:"6126d7a7",105:"8a927dcc",106:"4da8b56a",107:"592f8cc1",108:"28ba7bd1",109:"f6464bc0",110:"db919392",111:"ea30ebd1",112:"8d988769",113:"fea77ca6",114:"06ac168e",115:"e29a1244",116:"4e0e3f0d",117:"69c062f8",118:"6ea2feaf",119:"0ae4e688",120:"e6acb003",121:"07f1898e",122:"d6c2c200",123:"2aeec99a",124:"1c5329df",125:"cb6a7230",126:"0f850372",127:"b8876961",128:"037347f4",129:"f23bb09f",130:"8089cc99",131:"f3256090",132:"c9712562",133:"a545ed72",134:"8c02aab3",135:"cde1bc28",136:"6c1b54b7",137:"dd0e2d10",138:"00dc4bde",139:"b4e3431e",140:"54f16243",141:"d8730765",142:"fb9bbf42",143:"f66b2f78",144:"d97235a2",145:"243b745a",146:"4cbf6601",147:"f146a973",148:"3a61264f",149:"c7675cca",150:"b96c3fd3",151:"e9c8f77e",152:"b29de3ff",153:"8db267a7",154:"c2bc6db2",157:"1f338186",158:"319f81c0"}[e]+".js"}function t(f){if(d[f])return d[f].exports;var a=d[f]={i:f,l:!1,exports:{}};return e[f].call(a.exports,a,a.exports,t),a.l=!0,a.exports}t.e=function(e){var f=[],a=c[e];if(0!==a)if(a)f.push(a[2]);else{var d=new Promise((function(f,d){a=c[e]=[f,d]}));f.push(a[2]=d);var b,n=document.createElement("script");n.charset="utf-8",n.timeout=120,t.nc&&n.setAttribute("nonce",t.nc),n.src=r(e);var o=new Error;b=function(f){n.onerror=n.onload=null,clearTimeout(u);var a=c[e];if(0!==a){if(a){var d=f&&("load"===f.type?"missing":f.type),b=f&&f.target&&f.target.src;o.message="Loading chunk "+e+" failed.\n("+d+": "+b+")",o.name="ChunkLoadError",o.type=d,o.request=b,a[1](o)}c[e]=void 0}};var u=setTimeout((function(){b({type:"timeout",target:n})}),12e4);n.onerror=n.onload=b,document.head.appendChild(n)}return Promise.all(f)},t.m=e,t.c=d,t.d=function(e,f,a){t.o(e,f)||Object.defineProperty(e,f,{enumerable:!0,get:a})},t.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},t.t=function(e,f){if(1&f&&(e=t(e)),8&f)return e;if(4&f&&"object"==typeof e&&e&&e.__esModule)return e;var a=Object.create(null);if(t.r(a),Object.defineProperty(a,"default",{enumerable:!0,value:e}),2&f&&"string"!=typeof e)for(var d in e)t.d(a,d,function(f){return e[f]}.bind(null,d));return a},t.n=function(e){var f=e&&e.__esModule?function(){return e.default}:function(){return e};return t.d(f,"a",f),f},t.o=function(e,f){return Object.prototype.hasOwnProperty.call(e,f)},t.p="/netsage-pipeline/",t.gca=function(e){return r(e={17896441:"19",38962575:"30",49703371:"41",50032990:"46",63503174:"57",96852331:"85","012f72bc":"3","0214960a":"4","02341ef2":"5","0271802f":"6","0327c8bf":"7","0381afdd":"8","05c6fd0e":"9","066f8364":"10","07faddd5":"11","080a21b4":"12","092c2d1d":"13","0b8e461a":"14","0dd3dba6":"15","12ba72b5":"16","12ecfc4a":"17","16fcefd4":"18","1cc43674":"20","1d038b8a":"21","1f3a3353":"22","21ee0bc4":"23","22b7278f":"24","22d86c11":"25","3179f6dc":"26","3309db24":"27","33f6b1c9":"28","358e6759":"29","38d1fe39":"31","3914abfa":"32","40b960a4":"33","41980e0e":"34","42e094bc":"35","42f0e773":"36","42f7ba93":"37","43862f47":"38","46a600e4":"39","480c5911":"40","49e74cdd":"42","4a74d8e8":"43","4ff24bf8":"44","4ffdf56a":"45","514d766a":"47","56776a50":"48","56a29d1a":"49","58d958f4":"50","5a20f281":"51","5b6c4cb7":"52","5c4beb71":"53","5ee17acf":"54","5ef24f1d":"55","6125e2e5":"56","63f5fe43":"58","64269ed8":"59","653602f1":"60","6546ef57":"61","69c1008e":"62","6a059f5b":"63","6c279c12":"64","6d913b5d":"65","6e2a1c72":"66","6e3166bc":"67","71e5fd50":"68","734dc415":"69","799b1578":"70","7b85b03c":"71","7c78db09":"72","7d0b7755":"73","7d380c4c":"74","832ca04e":"75","83dbc9a5":"76","86d09e6e":"77","879923b2":"78","8997819f":"79","8b301107":"80","8bbfcfa9":"81","906738b8":"82","9191c864":"83","935f2afb":"84","9720d299":"86","97edd56e":"87","992819d4":"88","99d6017e":"89","9a5c9747":"90","9b505a1f":"91","9bfdba8c":"92","9d5eac8e":"93","9f628d52":"94","9fe55709":"95","9feaf8ab":"96",a04bc459:"97",a12215ea:"98",a55d7fc8:"99",a7459da2:"100",aa98ec4a:"101",aafd28b2:"102",abbf3a21:"103",ac3b7d69:"104",ad308aff:"105",ae8a1364:"106",b1bfe9f5:"107",b3c114f4:"108",b83a1049:"109",ba55e963:"110",bb237b1e:"111",bd6c6f88:"112",bed7ab4e:"113",c0bd86e8:"114",c3f49368:"115",c4f5d8e4:"116",c51f4c72:"117",cbd1c0d0:"118",cd902920:"119",ceb45c47:"120",d1597237:"121",d2c4f39e:"122",d3556f45:"123",d48b255e:"124",d5087a67:"125",d5140579:"126",d5479f42:"127",d71ff009:"128",d74c151a:"129",d75898ab:"130",d91f9def:"131",db0ed613:"132",dd1a9f26:"133",e00e17a0:"134",e10970d8:"135",e186625e:"136",e2be698d:"137",e2c161d9:"138",e30bbbcf:"139",e436ce66:"140",e746408e:"141",e7670743:"142",e987ec87:"143",ebfaece6:"144",f0b0282f:"145",f1ddb0ed:"146",f2cce206:"147",f485eca7:"148",f52ba0ac:"149",f64b3473:"150",f8bc0717:"151",f9995f72:"152",fa16f0fc:"153",ff1d9e71:"154"}[e]||e)},t.oe=function(e){throw console.error(e),e};var n=window.webpackJsonp=window.webpackJsonp||[],o=n.push.bind(n);n.push=f,n=n.slice();for(var u=0;u<n.length;u++)f(n[u]);var i=o;a()}([]);
\ No newline at end of file
diff --git a/assets/js/runtime~main.6b5de359.js b/assets/js/runtime~main.6b5de359.js
deleted file mode 100644
index dffcf468..00000000
--- a/assets/js/runtime~main.6b5de359.js
+++ /dev/null
@@ -1 +0,0 @@
-!function(e){function f(f){for(var c,r,t=f[0],n=f[1],o=f[2],u=0,l=[];u<t.length;u++)r=t[u],Object.prototype.hasOwnProperty.call(d,r)&&d[r]&&l.push(d[r][0]),d[r]=0;for(c in n)Object.prototype.hasOwnProperty.call(n,c)&&(e[c]=n[c]);for(i&&i(f);l.length;)l.shift()();return b.push.apply(b,o||[]),a()}function a(){for(var e,f=0;f<b.length;f++){for(var a=b[f],c=!0,r=1;r<a.length;r++){var n=a[r];0!==d[n]&&(c=!1)}c&&(b.splice(f--,1),e=t(t.s=a[0]))}return e}var c={},d={160:0},b=[];function r(e){return t.p+"assets/js/"+({3:"00cb704f",4:"012f72bc",5:"0214960a",6:"02341ef2",7:"0271802f",8:"0327c8bf",9:"0381afdd",10:"05c6fd0e",11:"066f8364",12:"07faddd5",13:"080a21b4",14:"0b8e461a",15:"0d6d53ac",16:"16fcefd4",17:"17896441",18:"1bbe794e",19:"1cc43674",20:"1d038b8a",21:"1f3a3353",22:"21ee0bc4",23:"22b7278f",24:"22d86c11",25:"3179f6dc",26:"3309db24",27:"33f6b1c9",28:"358e6759",29:"38962575",30:"38d1fe39",31:"3914abfa",32:"40b960a4",33:"418618b7",34:"41980e0e",35:"42e094bc",36:"42f0e773",37:"42f7ba93",38:"43862f47",39:"46a600e4",40:"480c5911",41:"49703371",42:"49e74cdd",43:"4a74d8e8",44:"4ff24bf8",45:"4ffdf56a",46:"50032990",47:"514d766a",48:"53b0b5d8",49:"56776a50",50:"56a29d1a",51:"58d958f4",52:"59305d05",53:"5a20f281",54:"5b6c4cb7",55:"5c4beb71",56:"5ee17acf",57:"6125e2e5",58:"63f5fe43",59:"64269ed8",60:"653602f1",61:"6546ef57",62:"66e31e6e",63:"69c1008e",64:"6a059f5b",65:"6c279c12",66:"6d913b5d",67:"734dc415",68:"799b1578",69:"7b85b03c",70:"7c78db09",71:"7d0b7755",72:"7d380c4c",73:"8084fbd4",74:"832ca04e",75:"83dbc9a5",76:"879923b2",77:"8997819f",78:"89d5514b",79:"8bbfcfa9",80:"9001d127",81:"906738b8",82:"935f2afb",83:"96852331",84:"9720d299",85:"97edd56e",86:"992819d4",87:"99d6017e",88:"9a5c9747",89:"9aa54785",90:"9b505a1f",91:"9b6d3056",92:"9bfdba8c",93:"9d5eac8e",94:"9e1ec80c",95:"9fe55709",96:"9feaf8ab",97:"a04bc459",98:"a12215ea",99:"a3c17fac",100:"a55d7fc8",101:"a7459da2",102:"aa98ec4a",103:"aafd28b2",104:"abbf3a21",105:"ac3b7d69",106:"ad308aff",107:"ae8a1364",108:"af9cfdf8",109:"b1bfe9f5",110:"b3c114f4",111:"b56f03c6",112:"b83a1049",113:"ba55e963",114:"bb237b1e",115:"bd6c6f88",116:"bed7ab4e",117:"c0bd86e8",118:"c3f49368",119:"c4f5d8e4",120:"c51f4c72",121:"cbd1c0d0",122:"cd902920",123:"ceb45c47",124:"d1597237",125:"d1b2eded",126:"d2c255b4",127:"d2c4f39e",128:"d3556f45",129:"d48b255e",130:"d5087a67",131:"d5140579",132:"d5479f42",133:"d71ff009",134:"d75898ab",135:"d91f9def",136:"db0ed613",137:"dd1a9f26",138:"ded423ee",139:"e00e17a0",140:"e10970d8",141:"e186625e",142:"e2be698d",143:"e2c161d9",144:"e30bbbcf",145:"e3649aa7",146:"e436ce66",147:"e4c61278",148:"e5ce8f83",149:"e746408e",150:"e7670743",151:"e987ec87",152:"ebfaece6",153:"f1ddb0ed",154:"f485eca7",155:"f52ba0ac",156:"f64b3473",157:"f8bc0717",158:"fa16f0fc"}[e]||e)+"."+{1:"bb3bc4d5",2:"89c52fba",3:"d8fb9eb4",4:"fac18353",5:"e4e7cd3e",6:"5a4f1f57",7:"b404da72",8:"c122f958",9:"426b41ab",10:"586b8e3b",11:"0e023e16",12:"24f5303f",13:"34495c8a",14:"4f5dc652",15:"2ae3b199",16:"2fe48b23",17:"1b171940",18:"484f5992",19:"8da06d41",20:"35918ded",21:"c8d7370b",22:"17500718",23:"fe1cf5a5",24:"0650c59f",25:"91efb769",26:"62c09093",27:"bd4095bc",28:"d9fbc6e9",29:"cec7a9ef",30:"6ca6725c",31:"09b700f7",32:"2c1c192b",33:"efff362b",34:"34ea6f7b",35:"2568d8ec",36:"55a7a6c7",37:"8e66f0a3",38:"792b1adb",39:"c1d369e6",40:"4a2f1d5a",41:"0ed6dd81",42:"e8f8d5a1",43:"964ee9cc",44:"5f1f44d6",45:"031ac316",46:"0ea61ccd",47:"b0ec51ab",48:"144adf6f",49:"28321f6f",50:"92eaa661",51:"6a5669a4",52:"7e6bc501",53:"847d72f0",54:"5d8d3771",55:"a62d61d9",56:"eeb64f93",57:"da94766a",58:"bed1e40c",59:"20920114",60:"c1abe2fd",61:"e13c6102",62:"ce62be57",63:"e9443d0d",64:"6c7d9748",65:"cc8d8222",66:"84d394c4",67:"80010e23",68:"a774f21c",69:"95489b8b",70:"bc8a300d",71:"4b75443f",72:"ad3c0101",73:"5592d29d",74:"54bca9dc",75:"e629373c",76:"2bf3c135",77:"64d4557b",78:"c223f8ef",79:"f0501de3",80:"1e177d17",81:"33482c0f",82:"5c9a98e4",83:"87e388ce",84:"d4ac0286",85:"3b71c751",86:"508016b4",87:"6764251a",88:"5a3e990f",89:"d3836c06",90:"0d4ca405",91:"ff7822d9",92:"baa4761b",93:"4b2aa660",94:"ab114838",95:"25f4e00b",96:"31cb6a73",97:"cf90e3e8",98:"35e2776f",99:"4f9a964b",100:"3675ba14",101:"03c88ff1",102:"ffbdd576",103:"21d293f7",104:"97ae394b",105:"fd3bbc4c",106:"9d4faea0",107:"d1aa581b",108:"a481c25f",109:"13251023",110:"e41fb427",111:"3836580d",112:"1cf10416",113:"5bed7840",114:"9e57bcc8",115:"e87a9bd6",116:"6b5e2485",117:"4043049b",118:"7fcf56c0",119:"76411a79",120:"0e8bec24",121:"5208c323",122:"a8ebc09f",123:"a3330b5f",124:"705b0ef9",125:"fdea14ce",126:"51d1c434",127:"b83f8cc2",128:"a168af5e",129:"d6a84f1e",130:"4b6f0656",131:"770e5402",132:"d126a913",133:"7f914194",134:"84f0767e",135:"b4c78c77",136:"02dc005f",137:"51ca4c1a",138:"df3ed6d9",139:"481c92d6",140:"6486e99c",141:"3f1e3436",142:"6ab0c41b",143:"57d50314",144:"e88d084c",145:"41406e34",146:"ffd97d6e",147:"03e947b6",148:"b18dcb47",149:"8663b558",150:"b2234f2e",151:"a4179a0d",152:"fbef59fb",153:"0d7a9793",154:"f3f2ea52",155:"ba4c9a10",156:"09cab9db",157:"28c2f6ec",158:"a192e853",161:"d19b2b8e",162:"71673972"}[e]+".js"}function t(f){if(c[f])return c[f].exports;var a=c[f]={i:f,l:!1,exports:{}};return e[f].call(a.exports,a,a.exports,t),a.l=!0,a.exports}t.e=function(e){var f=[],a=d[e];if(0!==a)if(a)f.push(a[2]);else{var c=new Promise((function(f,c){a=d[e]=[f,c]}));f.push(a[2]=c);var b,n=document.createElement("script");n.charset="utf-8",n.timeout=120,t.nc&&n.setAttribute("nonce",t.nc),n.src=r(e);var o=new Error;b=function(f){n.onerror=n.onload=null,clearTimeout(u);var a=d[e];if(0!==a){if(a){var c=f&&("load"===f.type?"missing":f.type),b=f&&f.target&&f.target.src;o.message="Loading chunk "+e+" failed.\n("+c+": "+b+")",o.name="ChunkLoadError",o.type=c,o.request=b,a[1](o)}d[e]=void 0}};var u=setTimeout((function(){b({type:"timeout",target:n})}),12e4);n.onerror=n.onload=b,document.head.appendChild(n)}return Promise.all(f)},t.m=e,t.c=c,t.d=function(e,f,a){t.o(e,f)||Object.defineProperty(e,f,{enumerable:!0,get:a})},t.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},t.t=function(e,f){if(1&f&&(e=t(e)),8&f)return e;if(4&f&&"object"==typeof e&&e&&e.__esModule)return e;var a=Object.create(null);if(t.r(a),Object.defineProperty(a,"default",{enumerable:!0,value:e}),2&f&&"string"!=typeof e)for(var c in e)t.d(a,c,function(f){return e[f]}.bind(null,c));return a},t.n=function(e){var f=e&&e.__esModule?function(){return e.default}:function(){return e};return t.d(f,"a",f),f},t.o=function(e,f){return Object.prototype.hasOwnProperty.call(e,f)},t.p="/netsage-pipeline/",t.gca=function(e){return r(e={17896441:"17",38962575:"29",49703371:"41",50032990:"46",96852331:"83","00cb704f":"3","012f72bc":"4","0214960a":"5","02341ef2":"6","0271802f":"7","0327c8bf":"8","0381afdd":"9","05c6fd0e":"10","066f8364":"11","07faddd5":"12","080a21b4":"13","0b8e461a":"14","0d6d53ac":"15","16fcefd4":"16","1bbe794e":"18","1cc43674":"19","1d038b8a":"20","1f3a3353":"21","21ee0bc4":"22","22b7278f":"23","22d86c11":"24","3179f6dc":"25","3309db24":"26","33f6b1c9":"27","358e6759":"28","38d1fe39":"30","3914abfa":"31","40b960a4":"32","418618b7":"33","41980e0e":"34","42e094bc":"35","42f0e773":"36","42f7ba93":"37","43862f47":"38","46a600e4":"39","480c5911":"40","49e74cdd":"42","4a74d8e8":"43","4ff24bf8":"44","4ffdf56a":"45","514d766a":"47","53b0b5d8":"48","56776a50":"49","56a29d1a":"50","58d958f4":"51","59305d05":"52","5a20f281":"53","5b6c4cb7":"54","5c4beb71":"55","5ee17acf":"56","6125e2e5":"57","63f5fe43":"58","64269ed8":"59","653602f1":"60","6546ef57":"61","66e31e6e":"62","69c1008e":"63","6a059f5b":"64","6c279c12":"65","6d913b5d":"66","734dc415":"67","799b1578":"68","7b85b03c":"69","7c78db09":"70","7d0b7755":"71","7d380c4c":"72","8084fbd4":"73","832ca04e":"74","83dbc9a5":"75","879923b2":"76","8997819f":"77","89d5514b":"78","8bbfcfa9":"79","9001d127":"80","906738b8":"81","935f2afb":"82","9720d299":"84","97edd56e":"85","992819d4":"86","99d6017e":"87","9a5c9747":"88","9aa54785":"89","9b505a1f":"90","9b6d3056":"91","9bfdba8c":"92","9d5eac8e":"93","9e1ec80c":"94","9fe55709":"95","9feaf8ab":"96",a04bc459:"97",a12215ea:"98",a3c17fac:"99",a55d7fc8:"100",a7459da2:"101",aa98ec4a:"102",aafd28b2:"103",abbf3a21:"104",ac3b7d69:"105",ad308aff:"106",ae8a1364:"107",af9cfdf8:"108",b1bfe9f5:"109",b3c114f4:"110",b56f03c6:"111",b83a1049:"112",ba55e963:"113",bb237b1e:"114",bd6c6f88:"115",bed7ab4e:"116",c0bd86e8:"117",c3f49368:"118",c4f5d8e4:"119",c51f4c72:"120",cbd1c0d0:"121",cd902920:"122",ceb45c47:"123",d1597237:"124",d1b2eded:"125",d2c255b4:"126",d2c4f39e:"127",d3556f45:"128",d48b255e:"129",d5087a67:"130",d5140579:"131",d5479f42:"132",d71ff009:"133",d75898ab:"134",d91f9def:"135",db0ed613:"136",dd1a9f26:"137",ded423ee:"138",e00e17a0:"139",e10970d8:"140",e186625e:"141",e2be698d:"142",e2c161d9:"143",e30bbbcf:"144",e3649aa7:"145",e436ce66:"146",e4c61278:"147",e5ce8f83:"148",e746408e:"149",e7670743:"150",e987ec87:"151",ebfaece6:"152",f1ddb0ed:"153",f485eca7:"154",f52ba0ac:"155",f64b3473:"156",f8bc0717:"157",fa16f0fc:"158"}[e]||e)},t.oe=function(e){throw console.error(e),e};var n=window.webpackJsonp=window.webpackJsonp||[],o=n.push.bind(n);n.push=f,n=n.slice();for(var u=0;u<n.length;u++)f(n[u]);var i=o;a()}([]);
\ No newline at end of file
diff --git a/assets/js/styles.19dac7a3.js b/assets/js/styles.ed63c98d.js
similarity index 77%
rename from assets/js/styles.19dac7a3.js
rename to assets/js/styles.ed63c98d.js
index e212bb73..0f7e7a33 100644
--- a/assets/js/styles.19dac7a3.js
+++ b/assets/js/styles.ed63c98d.js
@@ -1 +1 @@
-(window.webpackJsonp=window.webpackJsonp||[]).push([[0],{18:function(e,n,o){},190:function(e,n,o){e.exports={heroBanner:"heroBanner_3P7f",buttons:"buttons_1r9m",features:"features_3azU",featureImage:"featureImage_ZtzX"}},47:function(e,n,o){},51:function(e,n,o){},52:function(e,n,o){},53:function(e,n,o){},56:function(e,n,o){e.exports={skipToContent:"skipToContent_1oUP"}},57:function(e,n,o){e.exports={announcementBar:"announcementBar_3WsW",announcementBarClose:"announcementBarClose_38nx",announcementBarContent:"announcementBarContent_3EUC",announcementBarCloseable:"announcementBarCloseable_3myR"}},58:function(e,n,o){e.exports={toggle:"toggle_71bT"}},59:function(e,n,o){e.exports={searchWrapper:"searchWrapper_3rmH"}},60:function(e,n,o){e.exports={themedImage:"themedImage_1VuW","themedImage--light":"themedImage--light_3UqQ","themedImage--dark":"themedImage--dark_hz6m"}},61:function(e,n,o){e.exports={displayOnlyInLargeViewport:"displayOnlyInLargeViewport_GrZ2",navbarHideable:"navbarHideable_2qcr",navbarHidden:"navbarHidden_3yey"}},62:function(e,n,o){e.exports={footerLogoLink:"footerLogoLink_MyFc"}},63:function(e,n,o){},64:function(e,n,o){},79:function(e,n,o){e.exports={lastUpdatedDate:"lastUpdatedDate_1WI_"}},80:function(e,n,o){e.exports={tableOfContents:"tableOfContents_35-E",docItemContainer:"docItemContainer_gpai"}},81:function(e,n,o){e.exports={iconEdit:"iconEdit_2_ui"}},82:function(e,n,o){e.exports={docTitle:"docTitle_3a4h",docItemContainer:"docItemContainer_33ec",docItemCol:"docItemCol_3FnS"}},84:function(e,n,o){e.exports={sidebar:"sidebar_15mo",sidebarWithHideableNavbar:"sidebarWithHideableNavbar_267A",sidebarHidden:"sidebarHidden_2kNb",sidebarLogo:"sidebarLogo_3h0W",menu:"menu_Bmed",menuLinkText:"menuLinkText_2aKo",menuWithAnnouncementBar:"menuWithAnnouncementBar_2WvA",collapseSidebarButton:"collapseSidebarButton_1CGd",collapseSidebarButtonIcon:"collapseSidebarButtonIcon_3E-R",sidebarMenuIcon:"sidebarMenuIcon_fgN0",sidebarMenuCloseIcon:"sidebarMenuCloseIcon_1lpH",menuLinkExternal:"menuLinkExternal_1OhN"}},85:function(e,n,o){e.exports={codeBlockContainer:"codeBlockContainer_K1bP",codeBlockContent:"codeBlockContent_hGly",codeBlockTitle:"codeBlockTitle_eoMF",codeBlock:"codeBlock_23N8",codeBlockWithTitle:"codeBlockWithTitle_2JqI",copyButton:"copyButton_Ue-o",codeBlockLines:"codeBlockLines_39YC"}},86:function(e,n,o){},87:function(e,n,o){e.exports={enhancedAnchor:"enhancedAnchor_2LWZ"}},88:function(e,n,o){e.exports={docPage:"docPage_31aa",docMainContainer:"docMainContainer_3ufF",docMainContainerEnhanced:"docMainContainerEnhanced_3NYZ",docSidebarContainer:"docSidebarContainer_3Kbt",docSidebarContainerHidden:"docSidebarContainerHidden_3pA8",collapsedDocSidebar:"collapsedDocSidebar_2JMH",expandSidebarButtonIcon:"expandSidebarButtonIcon_1naQ",docItemWrapperEnhanced:"docItemWrapperEnhanced_2vyJ",docItemWrapper:"docItemWrapper_3FMP"}}}]);
\ No newline at end of file
+(window.webpackJsonp=window.webpackJsonp||[]).push([[0],{18:function(e,n,o){},187:function(e,n,o){e.exports={heroBanner:"heroBanner_3P7f",buttons:"buttons_1r9m",features:"features_3azU",featureImage:"featureImage_ZtzX"}},47:function(e,n,o){},51:function(e,n,o){},52:function(e,n,o){},53:function(e,n,o){},56:function(e,n,o){e.exports={skipToContent:"skipToContent_1oUP"}},57:function(e,n,o){e.exports={announcementBar:"announcementBar_3WsW",announcementBarClose:"announcementBarClose_38nx",announcementBarContent:"announcementBarContent_3EUC",announcementBarCloseable:"announcementBarCloseable_3myR"}},58:function(e,n,o){e.exports={toggle:"toggle_71bT"}},59:function(e,n,o){e.exports={searchWrapper:"searchWrapper_3rmH"}},60:function(e,n,o){e.exports={themedImage:"themedImage_1VuW","themedImage--light":"themedImage--light_3UqQ","themedImage--dark":"themedImage--dark_hz6m"}},61:function(e,n,o){e.exports={displayOnlyInLargeViewport:"displayOnlyInLargeViewport_GrZ2",navbarHideable:"navbarHideable_2qcr",navbarHidden:"navbarHidden_3yey"}},62:function(e,n,o){e.exports={footerLogoLink:"footerLogoLink_MyFc"}},63:function(e,n,o){},64:function(e,n,o){},81:function(e,n,o){e.exports={lastUpdatedDate:"lastUpdatedDate_1WI_"}},82:function(e,n,o){e.exports={tableOfContents:"tableOfContents_35-E",docItemContainer:"docItemContainer_gpai"}},83:function(e,n,o){e.exports={iconEdit:"iconEdit_2_ui"}},84:function(e,n,o){e.exports={docTitle:"docTitle_3a4h",docItemContainer:"docItemContainer_33ec",docItemCol:"docItemCol_3FnS"}},85:function(e,n,o){e.exports={sidebar:"sidebar_15mo",sidebarWithHideableNavbar:"sidebarWithHideableNavbar_267A",sidebarHidden:"sidebarHidden_2kNb",sidebarLogo:"sidebarLogo_3h0W",menu:"menu_Bmed",menuLinkText:"menuLinkText_2aKo",menuWithAnnouncementBar:"menuWithAnnouncementBar_2WvA",collapseSidebarButton:"collapseSidebarButton_1CGd",collapseSidebarButtonIcon:"collapseSidebarButtonIcon_3E-R",sidebarMenuIcon:"sidebarMenuIcon_fgN0",sidebarMenuCloseIcon:"sidebarMenuCloseIcon_1lpH",menuLinkExternal:"menuLinkExternal_1OhN"}},86:function(e,n,o){e.exports={codeBlockContainer:"codeBlockContainer_K1bP",codeBlockContent:"codeBlockContent_hGly",codeBlockTitle:"codeBlockTitle_eoMF",codeBlock:"codeBlock_23N8",codeBlockWithTitle:"codeBlockWithTitle_2JqI",copyButton:"copyButton_Ue-o",codeBlockLines:"codeBlockLines_39YC"}},87:function(e,n,o){},88:function(e,n,o){e.exports={enhancedAnchor:"enhancedAnchor_2LWZ"}},89:function(e,n,o){e.exports={docPage:"docPage_31aa",docMainContainer:"docMainContainer_3ufF",docMainContainerEnhanced:"docMainContainerEnhanced_3NYZ",docSidebarContainer:"docSidebarContainer_3Kbt",docSidebarContainerHidden:"docSidebarContainerHidden_3pA8",collapsedDocSidebar:"collapsedDocSidebar_2JMH",expandSidebarButtonIcon:"expandSidebarButtonIcon_1naQ",docItemWrapperEnhanced:"docItemWrapperEnhanced_2vyJ",docItemWrapper:"docItemWrapper_3FMP"}}}]);
\ No newline at end of file
diff --git a/docs/1.2.10/components/docker_env/index.html b/docs/1.2.10/components/docker_env/index.html
index d488b2e8..3a0f78fd 100644
--- a/docs/1.2.10/components/docker_env/index.html
+++ b/docs/1.2.10/components/docker_env/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Please copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Please copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e10970d8.6486e99c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e10970d8.cde1bc28.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;RNDNet New York Sflow&quot; and &quot;RNDNet Boston Netflow&quot; or &quot;RNDNet New York - London 1&quot; and &quot;RNDNet New York - London 2&quot;. Whatever makes sense in your situation.</p></div></div><ul><li>If you don&#x27;t set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. </li><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><p>Other settings of note in this file include the following. You will not necessarily need to change these, but be aware.</p><p><strong>rabbit_output_host</strong>: this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on <code>rabbit</code>, ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). (For NetSage, another logstash pipeline on a remote server moves flows from this final rabbit queue into Elasticsearch.)</p><p>The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.
 (See comments in the *-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
-on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e10970d8.6486e99c.js"></script>
+on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e10970d8.cde1bc28.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/components/docker_first_steps/index.html b/docs/1.2.10/components/docker_first_steps/index.html
index 5a9fec93..35b2af3f 100644
--- a/docs/1.2.10/components/docker_first_steps/index.html
+++ b/docs/1.2.10/components/docker_first_steps/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ae8a1364.d1aa581b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ae8a1364.4da8b56a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ae8a1364.d1aa581b.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ae8a1364.4da8b56a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/components/docker_pipeline/index.html b/docs/1.2.10/components/docker_pipeline/index.html
index 50f044aa..c506b1da 100644
--- a/docs/1.2.10/components/docker_pipeline/index.html
+++ b/docs/1.2.10/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/46a600e4.c1d369e6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/46a600e4.790fa3f6.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/46a600e4.c1d369e6.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/46a600e4.790fa3f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/bare_metal_install/index.html b/docs/1.2.10/deploy/bare_metal_install/index.html
index df3d3be9..7823f007 100644
--- a/docs/1.2.10/deploy/bare_metal_install/index.html
+++ b/docs/1.2.10/deploy/bare_metal_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Manual Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Manual Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e7670743.b2234f2e.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e7670743.fb9bbf42.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
 The Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. </p><p>Nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work.
 Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included.</p></div></div><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -41,16 +41,16 @@
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e7670743.b2234f2e.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e7670743.fb9bbf42.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/choose_install/index.html b/docs/1.2.10/deploy/choose_install/index.html
index 2e6c2622..8ec4de10 100644
--- a/docs/1.2.10/deploy/choose_install/index.html
+++ b/docs/1.2.10/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing an Installation Procedure | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Choosing an Installation Procedure | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Manual or BareMetal Installation"><meta data-react-helmet="true" property="og:description" content="Manual or BareMetal Installation"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/8bbfcfa9.f0501de3.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/8bbfcfa9.dc015f8e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/8bbfcfa9.f0501de3.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/8bbfcfa9.dc015f8e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/docker_install_advanced/index.html b/docs/1.2.10/deploy/docker_install_advanced/index.html
index fa8d611c..38797307 100644
--- a/docs/1.2.10/deploy/docker_install_advanced/index.html
+++ b/docs/1.2.10/deploy/docker_install_advanced/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Options Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Options Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><meta data-react-helmet="true" property="og:description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/f1ddb0ed.0d7a9793.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f1ddb0ed.4cbf6601.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
 router should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change the last part of the path to something meaningful.</li></ul><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h3><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; it will be replaced with a value set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h3><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=Example New York sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h3><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but you only want to keep some of them (eg, only send some of them to NetSage), use this option. The collectors and importer will process all flows, but in the logstash pipeline, those that do not have src_ifindex or dst_inindex equal to one of the listed interfaces will be dropped. </p><p>In the .env file, uncomment the apprpriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and list all the ifindex values of flows that should be kept and passed on to NetSage. You may enter one or more ifindex values. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, only flows that have src_ifindex = 123 or src_ifindex = 456 or dst_ifindex = 123 or dst_ifindex = 456 will be kept. All others will be dropped.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that come through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id in ElasticSearch) changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk don&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">7</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If you need to modify the amount of memory logstash can use or any other java settings,
 rename the provided example for JVM Options and tweak the settings as desired.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="for-data-saved-to-an-nfs-volume"></a>For Data Saved to an NFS Volume<a class="hash-link" href="#for-data-saved-to-an-nfs-volume" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either</p><ol><li>export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)</li><li>update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml</li></ol><p>Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values.</p><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#a-edit-docker-composeoverrideyml" class="table-of-contents__link">a. Edit docker-compose.override.yml</a></li><li><a href="#b--edit-netsage_overridexml" class="table-of-contents__link">b.  Edit netsage_override.xml</a></li><li><a href="#c-edit-environment-file" class="table-of-contents__link">c. Edit environment file</a></li><li><a href="#d-running-the-new-collector" class="table-of-contents__link">d. Running the new collector</a></li></ul></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#for-data-saved-to-an-nfs-volume" class="table-of-contents__link">For Data Saved to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/f1ddb0ed.0d7a9793.js"></script>
+You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#a-edit-docker-composeoverrideyml" class="table-of-contents__link">a. Edit docker-compose.override.yml</a></li><li><a href="#b--edit-netsage_overridexml" class="table-of-contents__link">b.  Edit netsage_override.xml</a></li><li><a href="#c-edit-environment-file" class="table-of-contents__link">c. Edit environment file</a></li><li><a href="#d-running-the-new-collector" class="table-of-contents__link">d. Running the new collector</a></li></ul></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#for-data-saved-to-an-nfs-volume" class="table-of-contents__link">For Data Saved to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f1ddb0ed.4cbf6601.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/docker_install_simple/index.html b/docs/1.2.10/deploy/docker_install_simple/index.html
index da91ca28..fbb5015d 100644
--- a/docs/1.2.10/deploy/docker_install_simple/index.html
+++ b/docs/1.2.10/deploy/docker_install_simple/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e746408e.8663b558.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e746408e.d8730765.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor (flow exporter/router), to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor (flow exporter/router), to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input rabbit queue &quot;netsage_deidentifier_raw&quot; on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-clone-the-netsage-pipeline-project"></a>2. Clone the Netsage Pipeline Project<a class="hash-link" href="#2-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t already, install <a href="https://www.docker.com" target="_blank" rel="noopener noreferrer">Docker</a> and <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">Docker Compose</a> and clone this project</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(If you are upgrading to a new release, see the Upgrade section below!)</p><p>Then checkout the right version of the code.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout {tag}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;{tag}&quot; with the release version you intend to use, e.g., &quot;v1.2.8&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.8.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-create-docker-composeoverrideyml"></a>3. Create Docker-compose.override.yml<a class="hash-link" href="#3-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you only have one collector, you should remove or comment out the section for the collector that is not used, so it doesn&#x27;t run and just create empty files.</p></div></div><p>This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to <code>/data/input_data/</code>. Each collector is namespaced by its type so the sflow collector will write data to <code>/data/input_data/sflow/</code> and the netflow collector will write data to <code>/data/input_data/netflow/</code>.  Change these only if required.</p><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-environment-file"></a>4. Create Environment File<a class="hash-link" href="#4-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
@@ -27,16 +27,16 @@
 (See comments in the *-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
 on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p><p></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which version Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier.
-This script will replace the version numbers of docker images in the docker-compose files with the correct values.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h2><p>After selecting the version to run, you could start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. </p><p>(Or see the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e746408e.8663b558.js"></script>
+This script will replace the version numbers of docker images in the docker-compose files with the correct values.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h2><p>After selecting the version to run, you could start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. </p><p>(Or see the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e746408e.d8730765.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/docker_troubleshoot/index.html b/docs/1.2.10/deploy/docker_troubleshoot/index.html
index 2b78ccf8..807ef9e3 100644
--- a/docs/1.2.10/deploy/docker_troubleshoot/index.html
+++ b/docs/1.2.10/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/cd902920.a8ebc09f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/cd902920.0ae4e688.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Logstash or Importer errors:</strong></p><ul><li>Make sure all containers are running.  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are comsuming. You need to add more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </li></ul><p>Please see the <a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/cd902920.a8ebc09f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Logstash or Importer errors:</strong></p><ul><li>Make sure all containers are running.  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are comsuming. You need to add more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </li></ul><p>Please see the <a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/cd902920.0ae4e688.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/deploy/docker_upgrade/index.html b/docs/1.2.10/deploy/docker_upgrade/index.html
index 2686c8c7..5367d930 100644
--- a/docs/1.2.10/deploy/docker_upgrade/index.html
+++ b/docs/1.2.10/deploy/docker_upgrade/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Upgrading | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Upgrading | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><meta data-react-helmet="true" property="og:description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/38962575.cec7a9ef.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/38962575.8edb151f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
 <code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-override-files"></a>Check/Update Override Files<a class="hash-link" href="#checkupdate-override-files" title="Direct link to heading">#</a></h3><p>Occasionally, the required version of Docker or nfdump may change, which will necessitate editing your override and/or env files.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code> to see if a new version of Docker is required. Look for, eg, <code>version: &quot;3.7&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.</p></li><li><p>Also check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:1.6.18</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.)
-Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.10). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this step!  Pull new images from Docker Hub. This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To save space, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-override-files" class="table-of-contents__link">Check/Update Override Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/38962575.cec7a9ef.js"></script>
+Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.10). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this step!  Pull new images from Docker Hub. This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To save space, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-override-files" class="table-of-contents__link">Check/Update Override Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/38962575.8edb151f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/devel/dev_dataset/index.html b/docs/1.2.10/devel/dev_dataset/index.html
index bf8ee389..a988ae13 100644
--- a/docs/1.2.10/devel/dev_dataset/index.html
+++ b/docs/1.2.10/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/f485eca7.f3f2ea52.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f485eca7.3a61264f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/f485eca7.f3f2ea52.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f485eca7.3a61264f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/devel/docker_dev_guide/index.html b/docs/1.2.10/devel/docker_dev_guide/index.html
index 79691243..f3af7c74 100644
--- a/docs/1.2.10/devel/docker_dev_guide/index.html
+++ b/docs/1.2.10/devel/docker_dev_guide/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Selecting a Version"><meta data-react-helmet="true" property="og:description" content="Selecting a Version"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/63f5fe43.bed1e40c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63f5fe43.faa7d00d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
 To select a released version use the docker_select_version.sh script (see the Deployment Guide).
 If you wish to use the development version (master branch) simply scip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-images"></a>Build Images<a class="hash-link" href="#build-images" title="Direct link to heading">#</a></h2><p>The images are published on Docker Hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in the GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/63f5fe43.bed1e40c.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/63f5fe43.faa7d00d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/devel/docker_dev_tag/index.html b/docs/1.2.10/devel/docker_dev_tag/index.html
index 3aaf148f..69aacdcc 100644
--- a/docs/1.2.10/devel/docker_dev_tag/index.html
+++ b/docs/1.2.10/devel/docker_dev_tag/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">How to Tag a New Release | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="How to Tag a New Release | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images."><meta data-react-helmet="true" property="og:description" content="To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/b83a1049.1cf10416.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/b83a1049.f6464bc0.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">How to Tag a New Release</h1></header><div class="markdown"><p>To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-releasetag"></a>In Github, Create a Release/Tag<a class="hash-link" href="#in-github-create-a-releasetag" title="Direct link to heading">#</a></h2><p>Be sure to copy info from the Changes file into the Release description.</p><p>Do this first ???</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-build-and-push-an-importer-image-manually"></a>To Build and Push an Importer Image Manually<a class="hash-link" href="#to-build-and-push-an-importer-image-manually" title="Direct link to heading">#</a></h2><p>Git clone the pipeline project and have the ?? branch checked out. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker push $image:$tag</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will build the image and push it to Docker Hub.</p><p>The person doing this has to have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level).</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="with-automation"></a>With Automation<a class="hash-link" href="#with-automation" title="Direct link to heading">#</a></h2><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="versioned-docs"></a>Versioned Docs<a class="hash-link" href="#versioned-docs" title="Direct link to heading">#</a></h2><p>A new set of versioned docs also has to be tagged. See the Docusaurus guide. </p><p>Does this have to happen before Building the image ??</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-nfdump"></a>New Version of Nfdump<a class="hash-link" href="#new-version-of-nfdump" title="Direct link to heading">#</a></h2><p>If a new version of dfdump has been released that we need,
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">How to Tag a New Release</h1></header><div class="markdown"><p>To tag a new release, first updated the version number and Changes file, build the rpm, etc.  and upgrade on bare-metal hosts using yum. If all works fine, do the following steps to create new Docker images.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-releasetag"></a>In Github, Create a Release/Tag<a class="hash-link" href="#in-github-create-a-releasetag" title="Direct link to heading">#</a></h2><p>Be sure to copy info from the Changes file into the Release description.</p><p>Do this first ???</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-build-and-push-an-importer-image-manually"></a>To Build and Push an Importer Image Manually<a class="hash-link" href="#to-build-and-push-an-importer-image-manually" title="Direct link to heading">#</a></h2><p>Git clone the pipeline project and have the ?? branch checked out. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker push $image:$tag</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will build the image and push it to Docker Hub.</p><p>The person doing this has to have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level).</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="with-automation"></a>With Automation<a class="hash-link" href="#with-automation" title="Direct link to heading">#</a></h2><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="versioned-docs"></a>Versioned Docs<a class="hash-link" href="#versioned-docs" title="Direct link to heading">#</a></h2><p>A new set of versioned docs also has to be tagged. See the Docusaurus guide. </p><p>Does this have to happen before Building the image ??</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-nfdump"></a>New Version of Nfdump<a class="hash-link" href="#new-version-of-nfdump" title="Direct link to heading">#</a></h2><p>If a new version of dfdump has been released that we need,
 ????</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-logstash"></a>New Version of Logstash<a class="hash-link" href="#new-version-of-logstash" title="Direct link to heading">#</a></h2><p>If a new version of logstash has been released that we want everyone to use,
-???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#in-github-create-a-releasetag" class="table-of-contents__link">In Github, Create a Release/Tag</a></li><li><a href="#to-build-and-push-an-importer-image-manually" class="table-of-contents__link">To Build and Push an Importer Image Manually</a></li><li><a href="#with-automation" class="table-of-contents__link">With Automation</a></li><li><a href="#versioned-docs" class="table-of-contents__link">Versioned Docs</a></li><li><a href="#new-version-of-nfdump" class="table-of-contents__link">New Version of Nfdump</a></li><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/b83a1049.1cf10416.js"></script>
+???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#in-github-create-a-releasetag" class="table-of-contents__link">In Github, Create a Release/Tag</a></li><li><a href="#to-build-and-push-an-importer-image-manually" class="table-of-contents__link">To Build and Push an Importer Image Manually</a></li><li><a href="#with-automation" class="table-of-contents__link">With Automation</a></li><li><a href="#versioned-docs" class="table-of-contents__link">Versioned Docs</a></li><li><a href="#new-version-of-nfdump" class="table-of-contents__link">New Version of Nfdump</a></li><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/b83a1049.f6464bc0.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/devel/docusaurus/index.html b/docs/1.2.10/devel/docusaurus/index.html
index a2e68fcd..a94deaff 100644
--- a/docs/1.2.10/devel/docusaurus/index.html
+++ b/docs/1.2.10/devel/docusaurus/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Revising Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Revising Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation uses Docusaurus."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation uses Docusaurus."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/514d766a.b0ec51ab.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/514d766a.ca35f79e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
 at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><p><em>THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-not-using-docker"></a>If Not Using Docker<a class="hash-link" href="#if-not-using-docker" title="Direct link to heading">#</a></h2><p>These are instructions for editing and releasing docs without using Docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><p>To get started the first time, install npm, then use that to install yarn </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo yum install npm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo npm install -g yarn </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd netsage-pipeline/website</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn install</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h3><p>If you are working on your local machine, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the build directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">go to http://localhost:3000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-changes"></a>To Make Changes<a class="hash-link" href="#to-make-changes" title="Direct link to heading">#</a></h3><p>Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn run docusaurus docs:version a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
@@ -28,16 +28,16 @@
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release-1"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release-1" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build build_docs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
 This will create new versioned docs in website/versioned_docs/.</p><p>Then edit docusaurus.config.js and change <code>lastVersion:</code> to refer to the new version number. </p><p>Finally, commit and push the following to github:</p><ul><li>website/versioned_docs/version-a.b.c/</li><li>website/versioned_sidebars/version-a.b.c.sidebars.json</li><li>versions.json </li><li>docusaurus.config.js</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="deploying-docs-to-githubio-1"></a>Deploying Docs to github.io<a class="hash-link" href="#deploying-docs-to-githubio-1" title="Direct link to heading">#</a></h3><p>How to do this when using Docker ??? Get into the container ???</p><p>For now, go a linux server that has yarn installed and
-follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Tag a New Release »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/514d766a.b0ec51ab.js"></script>
+follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Tag a New Release »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/514d766a.ca35f79e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/elastic/index.html b/docs/1.2.10/pipeline/elastic/index.html
index 8f1e60cd..77fe18ae 100644
--- a/docs/1.2.10/pipeline/elastic/index.html
+++ b/docs/1.2.10/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/4a74d8e8.964ee9cc.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/4a74d8e8.d930cabd.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/4a74d8e8.964ee9cc.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/4a74d8e8.d930cabd.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/importer/index.html b/docs/1.2.10/pipeline/importer/index.html
index b7777535..61277796 100644
--- a/docs/1.2.10/pipeline/importer/index.html
+++ b/docs/1.2.10/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/799b1578.a774f21c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/799b1578.ad714f78.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/799b1578.a774f21c.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/799b1578.ad714f78.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/intro/index.html b/docs/1.2.10/pipeline/intro/index.html
index f7af97d2..805ed806 100644
--- a/docs/1.2.10/pipeline/intro/index.html
+++ b/docs/1.2.10/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/49e74cdd.e8f8d5a1.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/49e74cdd.00e19770.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/49e74cdd.e8f8d5a1.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/49e74cdd.00e19770.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/logstash/index.html b/docs/1.2.10/pipeline/logstash/index.html
index 5d24a729..410959f5 100644
--- a/docs/1.2.10/pipeline/logstash/index.html
+++ b/docs/1.2.10/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0271802f.b404da72.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0271802f.e2ca0056.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 does some data type conversions;
@@ -28,16 +28,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0271802f.b404da72.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0271802f.e2ca0056.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/nfdump/index.html b/docs/1.2.10/pipeline/nfdump/index.html
index a8d3d231..57fd8526 100644
--- a/docs/1.2.10/pipeline/nfdump/index.html
+++ b/docs/1.2.10/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Sflow/Netflow Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Sflow/Netflow Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/f8bc0717.28c2f6ec.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f8bc0717.e9c8f77e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/1.2.10/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/f8bc0717.28c2f6ec.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/1.2.10/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f8bc0717.e9c8f77e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.10/pipeline/tstat/index.html b/docs/1.2.10/pipeline/tstat/index.html
index e48ef1bd..72ec597c 100644
--- a/docs/1.2.10/pipeline/tstat/index.html
+++ b/docs/1.2.10/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.10"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.10"><meta data-react-helmet="true" property="og:title" content="Tstat Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage GitHub Project"><meta data-react-helmet="true" property="og:description" content="Netsage GitHub Project"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0b8e461a.4f5dc652.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0b8e461a.4a1a6155.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d5087a67.4b6f0656.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0b8e461a.4f5dc652.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">Docker Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">Taggin a Release</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.10</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.10</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.10/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d5087a67.cb6a7230.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0b8e461a.4a1a6155.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/components/docker_env/index.html b/docs/1.2.11/components/docker_env/index.html
index f5f80e93..535dbe14 100644
--- a/docs/1.2.11/components/docker_env/index.html
+++ b/docs/1.2.11/components/docker_env/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Next, copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Next, copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/8997819f.64d4557b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/8997819f.aa7254b4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
-For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/8997819f.64d4557b.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
+For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/8997819f.aa7254b4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/components/docker_first_steps/index.html b/docs/1.2.11/components/docker_first_steps/index.html
index 04959f1e..40b2c2ef 100644
--- a/docs/1.2.11/components/docker_first_steps/index.html
+++ b/docs/1.2.11/components/docker_first_steps/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e987ec87.a4179a0d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e987ec87.f66b2f78.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e987ec87.a4179a0d.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e987ec87.f66b2f78.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/components/docker_pipeline/index.html b/docs/1.2.11/components/docker_pipeline/index.html
index 9e8eecdc..2d2e92b5 100644
--- a/docs/1.2.11/components/docker_pipeline/index.html
+++ b/docs/1.2.11/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/69c1008e.e9443d0d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/69c1008e.6660742f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/69c1008e.e9443d0d.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/69c1008e.6660742f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/bare_metal_install/index.html b/docs/1.2.11/deploy/bare_metal_install/index.html
index 33e0813b..e22bb16d 100644
--- a/docs/1.2.11/deploy/bare_metal_install/index.html
+++ b/docs/1.2.11/deploy/bare_metal_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Manual Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Manual Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0327c8bf.c122f958.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0327c8bf.4ed718d8.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
 The Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. </p><p>Nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work.
 Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included.</p></div></div><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -41,16 +41,16 @@
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0327c8bf.c122f958.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0327c8bf.4ed718d8.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/choose_install/index.html b/docs/1.2.11/deploy/choose_install/index.html
index 600e724e..4177ef7d 100644
--- a/docs/1.2.11/deploy/choose_install/index.html
+++ b/docs/1.2.11/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing an Installation Procedure | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Choosing an Installation Procedure | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Manual or BareMetal Installation"><meta data-react-helmet="true" property="og:description" content="Manual or BareMetal Installation"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/05c6fd0e.586b8e3b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/05c6fd0e.27777f60.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/05c6fd0e.586b8e3b.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/05c6fd0e.27777f60.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/docker_install_advanced/index.html b/docs/1.2.11/deploy/docker_install_advanced/index.html
index 35021146..1b4a798f 100644
--- a/docs/1.2.11/deploy/docker_install_advanced/index.html
+++ b/docs/1.2.11/deploy/docker_install_advanced/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Options Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Options Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><meta data-react-helmet="true" property="og:description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/22d86c11.0650c59f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/22d86c11.2a981864.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
 In the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.
 In the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.
 In the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  </p><p>Spaces don&#x27;t matter except within the sensor names. Punctuation is required as shown.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk doesn&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-save-flow-data-to-a-different-location"></a>To Save Flow Data to a Different Location<a class="hash-link" href="#to-save-flow-data-to-a-different-location" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options.</p><ol><li>The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.</li></ol><p>During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline dir}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir /var/netsage</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rm data/.placeholder</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rmdir data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ln -s /var/netsage {netsage-pipeline dir}/data</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(Check the permissions of the directory.)</p><ol start="2"><li>Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to <code>- /mydir/input_data/netflow:/data</code> (similarly for sflow) and set the importer and logstash volumes to <code>- /mydir:/data</code>. </li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/22d86c11.0650c59f.js"></script>
+You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/22d86c11.2a981864.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/docker_install_simple/index.html b/docs/1.2.11/deploy/docker_install_simple/index.html
index 4fb56483..d0f9075b 100644
--- a/docs/1.2.11/deploy/docker_install_simple/index.html
+++ b/docs/1.2.11/deploy/docker_install_simple/index.html
@@ -6,36 +6,36 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/dd1a9f26.51ca4c1a.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/dd1a9f26.a545ed72.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there will be <em>collectors</em> (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input rabbit queue &quot;netsage_deidentifier_raw&quot; on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-set-up-a-pipeline-host"></a>2. Set up a Pipeline Host<a class="hash-link" href="#2-set-up-a-pipeline-host" title="Direct link to heading">#</a></h3><p>Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. </p><p>Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at <a href="https://docs.docker.com/engine/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/engine/install/</a>.</p><p>Install Docker Compose from Docker&#x27;s GitHub repository - see <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/install/</a>.  You need to <strong>specify version 1.29.2</strong> (or newer) in the curl command. </p><p>Check default file permissions. If the <em>logstash</em> user is not able to access the logstash config files in the git checkout, you&#x27;ll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-clone-the-netsage-pipeline-project"></a>3. Clone the Netsage Pipeline Project<a class="hash-link" href="#3-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>Clone the netsage-pipeline project from github.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version.</p><p>Move into the netsage-pipeline/ directory (<strong>all git and docker commands must be run from inside this directory!</strong>), then checkout the most recent version of the code. It will say you are in &#x27;detached HEAD&#x27; state if you don&#x27;t include -b.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout {tag}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;{tag}&quot; with the release version you intend to use, e.g., &quot;v1.2.11&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.11.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-docker-composeoverrideyml"></a>4. Create Docker-compose.override.yml<a class="hash-link" href="#4-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. </p><ul><li>If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</li><li>If the collectors need to listen to different ports, make the appropriate changes here in both the &quot;command:&quot; and &quot;ports:&quot; lines. </li><li>By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in <code>./data/input_data/</code> (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.</li></ul><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem.</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which image versions Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier. </p><p>This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="6-create-environment-file"></a>6. Create Environment File<a class="hash-link" href="#6-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="testing-the-collectors"></a>Testing the Collectors<a class="hash-link" href="#testing-the-collectors" title="Direct link to heading">#</a></h2><p>At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  </p><p>(See the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Subdirectories for sflow/netflow, year, month, and day are created automatically under <code>data/input_data/</code>. File names contain dates and times.
 These are not text files; to view the contents, use an <a href="http://www.linuxcertif.com/man/1/nfdump/" target="_blank" rel="noopener noreferrer">nfdump command</a> (you will need to install nfdump).
 Files will be deleted automatically by the importer as they age out (the default is to keep 3 days).  </p><p>If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)<br>
-See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/dd1a9f26.51ca4c1a.js"></script>
+See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/dd1a9f26.a545ed72.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/docker_troubleshoot/index.html b/docs/1.2.11/deploy/docker_troubleshoot/index.html
index b6c7150c..25d2ee18 100644
--- a/docs/1.2.11/deploy/docker_troubleshoot/index.html
+++ b/docs/1.2.11/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7c78db09.bc8a300d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7c78db09.5ea6e962.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/7c78db09.bc8a300d.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/7c78db09.5ea6e962.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/deploy/docker_upgrade/index.html b/docs/1.2.11/deploy/docker_upgrade/index.html
index ee5a99fa..b4e4fa47 100644
--- a/docs/1.2.11/deploy/docker_upgrade/index.html
+++ b/docs/1.2.11/deploy/docker_upgrade/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Upgrading | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Upgrading | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><meta data-react-helmet="true" property="og:description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d48b255e.d6a84f1e.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d48b255e.1c5329df.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these three commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the conctainers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d48b255e.d6a84f1e.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these three commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the conctainers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d48b255e.1c5329df.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/devel/dev_dataset/index.html b/docs/1.2.11/devel/dev_dataset/index.html
index 1d536564..3dda1645 100644
--- a/docs/1.2.11/devel/dev_dataset/index.html
+++ b/docs/1.2.11/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9a5c9747.5a3e990f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9a5c9747.85ad9bca.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9a5c9747.5a3e990f.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9a5c9747.85ad9bca.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/devel/docker_dev_guide/index.html b/docs/1.2.11/devel/docker_dev_guide/index.html
index db9bd050..cff0c9a3 100644
--- a/docs/1.2.11/devel/docker_dev_guide/index.html
+++ b/docs/1.2.11/devel/docker_dev_guide/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Selecting a Version"><meta data-react-helmet="true" property="og:description" content="Selecting a Version"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/41980e0e.34ea6f7b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/41980e0e.d717c996.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
 To select a released version use the docker_select_version.sh script (see the Deployment Guide).
 If you wish to use the development version (master branch) simply skip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-images"></a>Build Images<a class="hash-link" href="#build-images" title="Direct link to heading">#</a></h2><p>The images are published on Docker Hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in the GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/41980e0e.34ea6f7b.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/41980e0e.d717c996.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/devel/docker_dev_tag/index.html b/docs/1.2.11/devel/docker_dev_tag/index.html
index bc617cf0..42c35ebe 100644
--- a/docs/1.2.11/devel/docker_dev_tag/index.html
+++ b/docs/1.2.11/devel/docker_dev_tag/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">How to Release a New Version of the Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="How to Release a New Version of the Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><meta data-react-helmet="true" property="og:description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6d913b5d.84d394c4.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6d913b5d.bb807553.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
 Be sure to copy info from the CHANGES file into the Release description.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-build-and-push-images-manually"></a>To Build and Push Images Manually<a class="hash-link" href="#to-build-and-push-images-manually" title="Direct link to heading">#</a></h2><p>Below is the procedure to build pipeline_importer and pipeline_logstash images manually.</p><p>Install docker-compose if not done already. See the Docker Installation instructions.</p><p>Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd netsage-pipeline</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b v1.2.11</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh 1.2.11</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     or  $ docker push $image:$tag                        (will push a specific image version)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.
 If that doesn&#x27;t work, try adding this to /etc/hosts: <code>67.219.148.138  mirrorlist.centos.org</code>, and/or try <code>yum install net-tools bridge-utils</code>, and/or restart network.service then docker. </p><p>The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level).</p><p>It might be a good idea to test the images before pushing them. See &quot;Test Docker Images&quot; below.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="building-with-automation"></a>Building With Automation<a class="hash-link" href="#building-with-automation" title="Direct link to heading">#</a></h2><p>???</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="test-docker-images"></a>Test Docker Images<a class="hash-link" href="#test-docker-images" title="Direct link to heading">#</a></h2><p>See the Docker installation instructions for details... </p><p>In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. </p><p>Run docker_select_version.sh if you haven&#x27;t already, then start it up <code>$ sudo docker-compose up -d</code>. If there are local images, they&#x27;ll be used, otherwise they&#x27;ll be pulled from Docker Hub.</p><p>After about 30 minutes, you should see flows in elasticsearch.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-versioned-docs"></a>Make Versioned Docs<a class="hash-link" href="#make-versioned-docs" title="Direct link to heading">#</a></h2><p>A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the <strong>Docusaurus guide</strong>. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-new-nfdump-collector-images"></a>To Make New Nfdump-Collector Images<a class="hash-link" href="#to-make-new-nfdump-collector-images" title="Direct link to heading">#</a></h2><p>If a new version of nfdump has been released that we need, new nfdump-collector images need to be made.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git clone https://github.com/netsage-project/docker-nfdump-collector.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd docker-nfdump-collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use squash: create a file at /etc/docker/daemon.json and put into it</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;experimental&quot;: true  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;debug: false&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To push to Docker Hub and quit docker</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker push netsage/nfdump-collector:$VER</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">...</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-logstash"></a>New Version of Logstash<a class="hash-link" href="#new-version-of-logstash" title="Direct link to heading">#</a></h3><p>If a new version of logstash has been released that we want everyone to use,
-???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/6d913b5d.84d394c4.js"></script>
+???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6d913b5d.bb807553.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/devel/docusaurus/index.html b/docs/1.2.11/devel/docusaurus/index.html
index af91e0a4..6703be9d 100644
--- a/docs/1.2.11/devel/docusaurus/index.html
+++ b/docs/1.2.11/devel/docusaurus/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Revising Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Revising Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation uses Docusaurus."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation uses Docusaurus."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9d5eac8e.4b2aa660.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9d5eac8e.6a3edbf9.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
 at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><p><em>THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-not-using-docker"></a>If Not Using Docker<a class="hash-link" href="#if-not-using-docker" title="Direct link to heading">#</a></h2><p>These are instructions for editing and releasing docs without using Docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><p>To get started the first time, install npm, then use that to install yarn </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo yum install npm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo npm install -g yarn </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd netsage-pipeline/website</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn install</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-local-development"></a>If Local Development<a class="hash-link" href="#if-local-development" title="Direct link to heading">#</a></h3><p>If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the build directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">go to http://localhost:3000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-changes"></a>To Make Changes<a class="hash-link" href="#to-make-changes" title="Direct link to heading">#</a></h3><p>Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><p>To view the changes you&#x27;ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the &quot;Deploying Docs to github.io&quot; section below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn run docusaurus docs:version a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
@@ -28,16 +28,16 @@
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release-1"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release-1" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build build_docs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
 This will create new versioned docs in website/versioned_docs/.</p><p>Then edit docusaurus.config.js and change <code>lastVersion:</code> to refer to the new version number. </p><p>Finally, commit and push the following to github:</p><ul><li>website/versioned_docs/version-a.b.c/</li><li>website/versioned_sidebars/version-a.b.c.sidebars.json</li><li>versions.json </li><li>docusaurus.config.js</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="deploying-docs-to-githubio-1"></a>Deploying Docs to github.io<a class="hash-link" href="#deploying-docs-to-githubio-1" title="Direct link to heading">#</a></h3><p>How to do this when using Docker ??? Get into the container ???</p><p>For now, go a linux server that has yarn installed and
-follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9d5eac8e.4b2aa660.js"></script>
+follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9d5eac8e.6a3edbf9.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/elastic/index.html b/docs/1.2.11/pipeline/elastic/index.html
index 72c6a87e..252be849 100644
--- a/docs/1.2.11/pipeline/elastic/index.html
+++ b/docs/1.2.11/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/5b6c4cb7.5d8d3771.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/5b6c4cb7.1e5700c2.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/5b6c4cb7.5d8d3771.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/5b6c4cb7.1e5700c2.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/importer/index.html b/docs/1.2.11/pipeline/importer/index.html
index 0187dde2..4d1273f9 100644
--- a/docs/1.2.11/pipeline/importer/index.html
+++ b/docs/1.2.11/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/96852331.87e388ce.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/96852331.822f904f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/96852331.87e388ce.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/96852331.822f904f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/intro/index.html b/docs/1.2.11/pipeline/intro/index.html
index b338ce43..8aa7fc20 100644
--- a/docs/1.2.11/pipeline/intro/index.html
+++ b/docs/1.2.11/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/a12215ea.35e2776f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/a12215ea.1ea3e4ef.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/a12215ea.35e2776f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/a12215ea.1ea3e4ef.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/logstash/index.html b/docs/1.2.11/pipeline/logstash/index.html
index 41e301af..a94e7419 100644
--- a/docs/1.2.11/pipeline/logstash/index.html
+++ b/docs/1.2.11/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/012f72bc.fac18353.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/012f72bc.36b7fe96.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 sets duration and rates to 0 if duration is &lt;= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="15-sensor-specific-changesconf"></a>15-sensor-specific-changes.conf<a class="hash-link" href="#15-sensor-specific-changesconf" title="Direct link to heading">#</a></h3><p>Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="20-add_idconf"></a>20-add_id.conf<a class="hash-link" href="#20-add_idconf" title="Direct link to heading">#</a></h3><p>Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="40-aggregationconf"></a>40-aggregation.conf<a class="hash-link" href="#40-aggregationconf" title="Direct link to heading">#</a></h3><p>Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow.</p><p>Notes: </p><ul><li>By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together.</li><li>If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes.</li><li>There is another &quot;timeout&quot; setting which is basically the maximum duration of a stitched flow (default: 24 hr).</li><li>When logstash shuts down, any flows &quot;in the aggregator&quot; will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. </li><li>Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file.</li><li>Tstat flows come in already complete, so no aggregation is done on those flows.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="45-geoip-taggingconf"></a>45-geoip-tagging.conf<a class="hash-link" href="#45-geoip-taggingconf" title="Direct link to heading">#</a></h3><p>Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;
@@ -27,16 +27,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/012f72bc.fac18353.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/012f72bc.36b7fe96.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/nfdump/index.html b/docs/1.2.11/pipeline/nfdump/index.html
index e47525f4..2036bfcc 100644
--- a/docs/1.2.11/pipeline/nfdump/index.html
+++ b/docs/1.2.11/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Sflow/Netflow Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Sflow/Netflow Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e30bbbcf.e88d084c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e30bbbcf.b4e3431e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/1.2.11/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e30bbbcf.e88d084c.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/1.2.11/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e30bbbcf.b4e3431e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.11/pipeline/tstat/index.html b/docs/1.2.11/pipeline/tstat/index.html
index 101852aa..8cf69c7a 100644
--- a/docs/1.2.11/pipeline/tstat/index.html
+++ b/docs/1.2.11/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.11"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.11"><meta data-react-helmet="true" property="og:title" content="Tstat Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage GitHub Project"><meta data-react-helmet="true" property="og:description" content="Netsage GitHub Project"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/f52ba0ac.ba4c9a10.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/02341ef2.76f0a379.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f52ba0ac.c7675cca.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/02341ef2.5a4f1f57.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/f52ba0ac.ba4c9a10.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.11</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.11</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.11/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/02341ef2.76f0a379.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f52ba0ac.c7675cca.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/deploy/choose_install/index.html b/docs/1.2.5/deploy/choose_install/index.html
index 66dc5157..7835f7d4 100644
--- a/docs/1.2.5/deploy/choose_install/index.html
+++ b/docs/1.2.5/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing Install | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Choosing Install | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="BareMetal"><meta data-react-helmet="true" property="og:description" content="BareMetal"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/3179f6dc.91efb769.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/3179f6dc.bedc2974.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal"></a>BareMetal<a class="hash-link" href="#baremetal" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker</a></li><li><a href="/netsage-pipeline/docs/1.2.5/deploy/install">BareMetal</a></li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/devel/docker"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Install Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal" class="table-of-contents__link">BareMetal</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/3179f6dc.91efb769.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal"></a>BareMetal<a class="hash-link" href="#baremetal" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker</a></li><li><a href="/netsage-pipeline/docs/1.2.5/deploy/install">BareMetal</a></li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/devel/docker"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Install Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal" class="table-of-contents__link">BareMetal</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/3179f6dc.bedc2974.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/deploy/docker_install/index.html b/docs/1.2.5/deploy/docker_install/index.html
index 64e1ee1b..4cb86c63 100644
--- a/docs/1.2.5/deploy/docker_install/index.html
+++ b/docs/1.2.5/deploy/docker_install/index.html
@@ -6,35 +6,35 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all"><meta data-react-helmet="true" property="og:description" content="The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/a04bc459.cf90e3e8.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/a04bc459.dcff4f83.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all
 the nuances of getting the pipeline setup.</p><p>Before we start, you may have a read over the <a href="/netsage-pipeline/docs/1.2.5/devel/docker">developer docker guide</a> it contains several notes such as how to select the docker version and likely other bits of information you may find useful.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>Nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h2><p>Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you&#x27;ll be defining.</p><p>More info of nfdump can be found <a href="https://github.com/phaag/nfdump/" target="_blank" rel="noopener noreferrer">here</a></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="external-nfdump"></a>External Nfdump<a class="hash-link" href="#external-nfdump" title="Direct link to heading">#</a></h3><p>In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided.</p><p>You&#x27;ll need to update your scripts to output to \$PROJECT/data/input_data. Naturally all the paths are configurable but you&#x27;ll have a much easier if you stick to the defaults.</p><p>If you do choose to store the data elsewhere, the location may still need to be inside of the \$PROJECT or a docker volume location in order for docker to be able to reference it.</p><p>You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data.</p><p>At this point please proceed to <a href="#common-pattern">Common Pattern</a></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-nfdump"></a>Dockerized Nfdump<a class="hash-link" href="#dockerized-nfdump" title="Direct link to heading">#</a></h3><p>After you&#x27;ve selected the version of docker you&#x27;ll be running. you can start the collector by simply running:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p><p>By default the container comes up and will write data to <code>./data/input_data</code> and listen to udp traffic on localhost:9999.</p><p>continue to <a href="#common-pattern">Common Pattern</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="common-pattern"></a>Common Pattern<a class="hash-link" href="#common-pattern" title="Direct link to heading">#</a></h2><p>Before continuing you need to choose if you are going to be use the &#x27;Develop&#x27; version which has the latest changes but might be a bit less stable or using the &#x27;Release&#x27; version.</p><ul><li><p>Development Version</p><ul><li>Disregard anything about <code>docker_select_version.sh</code> that will not apply to you </li><li>Update to latest code git pull origin master</li></ul></li><li><p>Release version</p><ul><li><code>git fetch; git checkout &lt;tag name&gt;</code> replace &quot;tag name&quot; with v1.2.5 or the version you intend to use.</li><li>Please select the version you wish to use using <code>./scripts/docker_select_version.sh</code>
 it is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.</li></ul></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment file<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>Please make a copy of the .env and refer back to the docker <a href="/netsage-pipeline/docs/1.2.5/devel/docker">dev guide</a> on details on configuring the env. Most of the default value should work just fine.</p><p>The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as.</p><p>If you don&#x27;t send a sensor name it&#x27;ll use the default docker hostname which changes each time you run the pipeline.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sensorName=bestSensorEver</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="bringing-up-the-pipeline"></a>Bringing up the Pipeline<a class="hash-link" href="#bringing-up-the-pipeline" title="Direct link to heading">#</a></h3><p>Starting up the pipeline using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You can check the logs for each of the container by running</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shutting-down-the-pipeline"></a>Shutting Down the pipeline.<a class="hash-link" href="#shutting-down-the-pipeline" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="kibana-and-elastic-search"></a>Kibana and Elastic Search<a class="hash-link" href="#kibana-and-elastic-search" title="Direct link to heading">#</a></h3><p>The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.5/devel/docker#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  hostname:9999 is the default.</li><li>Make sure you created a .env file and updated the settings accordingly.</li><li>sensorName especially since that identifies the source of the data. </li><li>check the logs of the various components to see if anything jumps out as being invalid.  docker-compose logs -f &lt;service_label&gt;</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory.</p><p>Applying this snippet to logstash may help. Naturally the values will have to change.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>If your only changes are the version you selected simply reset and discard your changes.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development"></a>Development<a class="hash-link" href="#development" title="Direct link to heading">#</a></h4><p>Update the git repo. Likely this won&#x27;t change anything but it&#x27;s always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="release"></a>Release<a class="hash-link" href="#release" title="Direct link to heading">#</a></h4><ol><li>git checkout &lt;tag_value&gt; (ie. v1.2.6, v1.2.7 etc)</li><li><code>./scripts/docker_select_version.sh</code> select the same version as the tag you checked out.</li></ol><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/docker_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Install Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#nfdump" class="table-of-contents__link">Nfdump</a><ul><li><a href="#external-nfdump" class="table-of-contents__link">External Nfdump</a></li><li><a href="#dockerized-nfdump" class="table-of-contents__link">Dockerized Nfdump</a></li></ul></li><li><a href="#common-pattern" class="table-of-contents__link">Common Pattern</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment file</a></li><li><a href="#bringing-up-the-pipeline" class="table-of-contents__link">Bringing up the Pipeline</a></li><li><a href="#shutting-down-the-pipeline" class="table-of-contents__link">Shutting Down the pipeline.</a></li><li><a href="#kibana-and-elastic-search" class="table-of-contents__link">Kibana and Elastic Search</a></li></ul></li><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a><ul><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/a04bc459.cf90e3e8.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>If your only changes are the version you selected simply reset and discard your changes.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development"></a>Development<a class="hash-link" href="#development" title="Direct link to heading">#</a></h4><p>Update the git repo. Likely this won&#x27;t change anything but it&#x27;s always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="release"></a>Release<a class="hash-link" href="#release" title="Direct link to heading">#</a></h4><ol><li>git checkout &lt;tag_value&gt; (ie. v1.2.6, v1.2.7 etc)</li><li><code>./scripts/docker_select_version.sh</code> select the same version as the tag you checked out.</li></ol><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/docker_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Install Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#nfdump" class="table-of-contents__link">Nfdump</a><ul><li><a href="#external-nfdump" class="table-of-contents__link">External Nfdump</a></li><li><a href="#dockerized-nfdump" class="table-of-contents__link">Dockerized Nfdump</a></li></ul></li><li><a href="#common-pattern" class="table-of-contents__link">Common Pattern</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment file</a></li><li><a href="#bringing-up-the-pipeline" class="table-of-contents__link">Bringing up the Pipeline</a></li><li><a href="#shutting-down-the-pipeline" class="table-of-contents__link">Shutting Down the pipeline.</a></li><li><a href="#kibana-and-elastic-search" class="table-of-contents__link">Kibana and Elastic Search</a></li></ul></li><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a><ul><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/a04bc459.dcff4f83.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/deploy/install/index.html b/docs/1.2.5/deploy/install/index.html
index f177dd63..dcdc1cb6 100644
--- a/docs/1.2.5/deploy/install/index.html
+++ b/docs/1.2.5/deploy/install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">NetSage Flow Processing Pipeline Install Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="NetSage Flow Processing Pipeline Install Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9fe55709.25f4e00b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9fe55709.e9c7bfda.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Install Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>Minimum components</p><ul><li>Data Injestion Source (nfdump, tstat or both)</li><li>RabbitMQ</li><li>LogStash</li><li>Importer (If you use nfdump)</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="prerequirements"></a>Prerequirements<a class="hash-link" href="#prerequirements" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed.</p><ol><li>nfdump </li><li>tstat</li></ol><p>You&#x27;ll need to have at least one of them setup in order to be able to process the data</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h3><p>The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>Once nfdump is setup you&#x27;ll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbitmq"></a>RabbitMQ<a class="hash-link" href="#rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">`</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Install Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>Minimum components</p><ul><li>Data Injestion Source (nfdump, tstat or both)</li><li>RabbitMQ</li><li>LogStash</li><li>Importer (If you use nfdump)</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="prerequirements"></a>Prerequirements<a class="hash-link" href="#prerequirements" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed.</p><ol><li>nfdump </li><li>tstat</li></ol><p>You&#x27;ll need to have at least one of them setup in order to be able to process the data</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h3><p>The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>Once nfdump is setup you&#x27;ll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbitmq"></a>RabbitMQ<a class="hash-link" href="#rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">`</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline"></a>Installing the Pipeline<a class="hash-link" href="#installing-the-pipeline" title="Direct link to heading">#</a></h2><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise.</li><li>Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them</li></ol><p>send directly to the appropriate rabbit queue. </p><ol start="3"><li>Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows </li></ol><p>that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.
 The final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the
 last logstash conf here to write to elasticsearch.</p><p>Nothing will automatically start after installation as we need to move on to configuration. Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p>Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files [THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]. This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: <code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code> </p><p>The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;!--  &quot;instance&quot; goes along with sensor</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       This is to identify various instances if a sensor has more than one &quot;stream&quot; / data collection</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       Defaults to 0.</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;instance&gt;1&lt;/instance&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">--&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       Defaults to sensor, but you can set it to something else here</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      &lt;router-address&gt;&lt;/router-address&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    Flow type: type of flow data (defaults to netflow)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;/collection&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Notice that <code>instance</code> , <code>router-address</code> , and <code>flow-type</code> are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file.</p><p>You can have multiple <code>collection</code> stanzas, to import multiple collections on one host.</p><p>The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you&#x27;re running a default RabbitMQ config, which is open only to &#x27;localhost&#x27; as guest/guest, you won&#x27;t need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;config&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files1&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname1.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files2&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname2.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -33,16 +33,16 @@
 Create logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">```sh</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">To list the keys:</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>  $ sudo -E logstash-keystore list</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">To remove a key-value pair:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">```sh</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore remove &lt;key name&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>FLOW STITCHING - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use &quot;-w 1&quot;.</p><p>See the comments in 04-stitching.conf to learn more about how complete flows are defined.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#prerequirements" class="table-of-contents__link">Prerequirements</a><ul><li><a href="#nfdump" class="table-of-contents__link">nfdump</a></li></ul></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#rabbitmq" class="table-of-contents__link">RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline" class="table-of-contents__link">Installing the Pipeline</a></li><li><a href="#configuration" class="table-of-contents__link">Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#configuring-the-pipeline-stages" class="table-of-contents__link">Configuring the Pipeline Stages</a></li><li><a href="#shared-config-file-listing" class="table-of-contents__link">Shared config file listing</a></li></ul></li><li><a href="#running-the-daemons" class="table-of-contents__link">Running the daemons</a><ul><li><a href="#daemon-listing" class="table-of-contents__link">Daemon Listing</a></li></ul></li><li><a href="#setup-notes" class="table-of-contents__link">Setup Notes</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9fe55709.25f4e00b.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use &quot;-w 1&quot;.</p><p>See the comments in 04-stitching.conf to learn more about how complete flows are defined.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/deploy/install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#prerequirements" class="table-of-contents__link">Prerequirements</a><ul><li><a href="#nfdump" class="table-of-contents__link">nfdump</a></li></ul></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#rabbitmq" class="table-of-contents__link">RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline" class="table-of-contents__link">Installing the Pipeline</a></li><li><a href="#configuration" class="table-of-contents__link">Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#configuring-the-pipeline-stages" class="table-of-contents__link">Configuring the Pipeline Stages</a></li><li><a href="#shared-config-file-listing" class="table-of-contents__link">Shared config file listing</a></li></ul></li><li><a href="#running-the-daemons" class="table-of-contents__link">Running the daemons</a><ul><li><a href="#daemon-listing" class="table-of-contents__link">Daemon Listing</a></li></ul></li><li><a href="#setup-notes" class="table-of-contents__link">Setup Notes</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9fe55709.e9c7bfda.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/devel/docker/index.html b/docs/1.2.5/devel/docker/index.html
index ca379be6..76634689 100644
--- a/docs/1.2.5/devel/docker/index.html
+++ b/docs/1.2.5/devel/docker/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Docker Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Docker Setup"><meta data-react-helmet="true" property="og:description" content="Docker Setup"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/83dbc9a5.e629373c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/83dbc9a5.4a4e5652.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Docker Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><p><code>scripts/docker_select_version.sh tagValue</code></p><p>Example:</p><p><code>scripts/docker_select_version.sh v1.2.5</code></p><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Docker Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><p><code>scripts/docker_select_version.sh tagValue</code></p><p>Example:</p><p><code>scripts/docker_select_version.sh v1.2.5</code></p><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
 any git checkout.  Simply follow the directions below and setup your pipeline as instructed.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-base-images"></a>Build Base Images<a class="hash-link" href="#build-base-images" title="Direct link to heading">#</a></h2><p>This is optional.  The image are published on docker hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in this GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuring-the-containers"></a>Configuring the Containers<a class="hash-link" href="#configuring-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t done so already, copy env.example and update it to match your own settings:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbit"></a>Rabbit<a class="hash-link" href="#rabbit" title="Direct link to heading">#</a></h3><p>This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set
 here should be consistent with the config for the logstash and importer </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_ERLANG_COOKIE=&#x27;secret cookie&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBIT_HOST=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_USER=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_PASS=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">discovery.type=single-node</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it&#x27;s set to rabbit</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ <strong>NOTE: Changes will require you to rebuild the container</strong></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h3><p>Define the input rabbit queue.  This should match the importer output queue</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line:  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Logstash</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/83dbc9a5.e629373c.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line:  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Logstash</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/83dbc9a5.4a4e5652.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/docusaurus/index.html b/docs/1.2.5/docusaurus/index.html
index 9a352dce..ae8a88f0 100644
--- a/docs/1.2.5/docusaurus/index.html
+++ b/docs/1.2.5/docusaurus/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Documentation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Documentation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/480c5911.4a2f1d5a.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/480c5911.48d5baca.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  </p><p>To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation
-at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/DOCUMENTATION.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/480c5911.4a2f1d5a.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  </p><p>To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation
+at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/DOCUMENTATION.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/480c5911.48d5baca.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/pipeline/index.html b/docs/1.2.5/pipeline/index.html
index 27d485a4..90e72c6c 100644
--- a/docs/1.2.5/pipeline/index.html
+++ b/docs/1.2.5/pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/c0bd86e8.4043049b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/c0bd86e8.06ac168e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Pipeline</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h1><p>The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>The Pipeline is made of the following components (Currently)</p><ul><li><a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">Importer</a>  (Collection of perl scripts)<ul><li><a href="/netsage-pipeline/docs/1.2.5/pipeline_importer">doc</a></li></ul></li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Elastic Logstash</a> Performs a variety of transformation on the data<ul><li><a href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">doc</a> </li></ul></li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a> used for message passing and queing of tasks.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sensors-and-data-collection"></a>Sensors and Data Collection<a class="hash-link" href="#sensors-and-data-collection" title="Direct link to heading">#</a></h2><p>&quot;Testpoints&quot; or &quot;sensors&quot; collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). </p><p>Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#sensors-and-data-collection" class="table-of-contents__link">Sensors and Data Collection</a><ul><li><a href="#importer" class="table-of-contents__link">Importer</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/c0bd86e8.4043049b.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Pipeline</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h1><p>The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>The Pipeline is made of the following components (Currently)</p><ul><li><a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">Importer</a>  (Collection of perl scripts)<ul><li><a href="/netsage-pipeline/docs/1.2.5/pipeline_importer">doc</a></li></ul></li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Elastic Logstash</a> Performs a variety of transformation on the data<ul><li><a href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">doc</a> </li></ul></li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a> used for message passing and queing of tasks.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sensors-and-data-collection"></a>Sensors and Data Collection<a class="hash-link" href="#sensors-and-data-collection" title="Direct link to heading">#</a></h2><p>&quot;Testpoints&quot; or &quot;sensors&quot; collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). </p><p>Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#sensors-and-data-collection" class="table-of-contents__link">Sensors and Data Collection</a><ul><li><a href="#importer" class="table-of-contents__link">Importer</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/c0bd86e8.06ac168e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/pipeline_importer/index.html b/docs/1.2.5/pipeline_importer/index.html
index 303192d2..21b3a6be 100644
--- a/docs/1.2.5/pipeline_importer/index.html
+++ b/docs/1.2.5/pipeline_importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Importer"><meta data-react-helmet="true" property="og:description" content="Importer"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/066f8364.0e023e16.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/066f8364.bfb615a4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-1"></a>Importer<a class="hash-link" href="#importer-1" title="Direct link to heading">#</a></h2><p>NOTE: Importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>configuration files for the importer are <code>netsage_netflow_importer.xml</code> and <code>netsage_shared.xml</code> in <code>/etc/grnoc/netsage/deidentifer/</code>. Comments in the files briefly describe the options.</p><p>Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="internals"></a>Internals<a class="hash-link" href="#internals" title="Direct link to heading">#</a></h3><p>The importer uses the nfdump command with -a to aggregate within the file, and -L <code>threshold</code> to throw out any flows under a flow size threshold. </p><p>For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.
-A ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages.</p><p>All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Logstash »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#importer-1" class="table-of-contents__link">Importer</a><ul><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li><li><a href="#internals" class="table-of-contents__link">Internals</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/066f8364.0e023e16.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-1"></a>Importer<a class="hash-link" href="#importer-1" title="Direct link to heading">#</a></h2><p>NOTE: Importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>configuration files for the importer are <code>netsage_netflow_importer.xml</code> and <code>netsage_shared.xml</code> in <code>/etc/grnoc/netsage/deidentifer/</code>. Comments in the files briefly describe the options.</p><p>Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="internals"></a>Internals<a class="hash-link" href="#internals" title="Direct link to heading">#</a></h3><p>The importer uses the nfdump command with -a to aggregate within the file, and -L <code>threshold</code> to throw out any flows under a flow size threshold. </p><p>For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.
+A ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages.</p><p>All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Logstash »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#importer-1" class="table-of-contents__link">Importer</a><ul><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li><li><a href="#internals" class="table-of-contents__link">Internals</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/066f8364.bfb615a4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.5/pipeline_logstash/index.html b/docs/1.2.5/pipeline_logstash/index.html
index 94f2b960..291da53b 100644
--- a/docs/1.2.5/pipeline_logstash/index.html
+++ b/docs/1.2.5/pipeline_logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Logstash | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.5"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.5"><meta data-react-helmet="true" property="og:title" content="Pipeline Logstash | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Logstash"><meta data-react-helmet="true" property="og:description" content="Logstash"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/a7459da2.03c88ff1.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6c279c12.225c0b15.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/a7459da2.2d033572.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Pipeline Logstash</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h1><p>These Logstash config files are in /etc/logstash/conf.d/</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-procedures"></a>Logstash Procedures<a class="hash-link" href="#logstash-procedures" title="Direct link to heading">#</a></h2><p>The following steps are defined for logstash:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-inputsconf"></a>01-inputs.conf<a class="hash-link" href="#01-inputsconf" title="Direct link to heading">#</a></h3><p>reads flows from wherever the user wants to get data; in our case, from the netsage_deidentifer_raw rabbit queue.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.5/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.5</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.5</span></div><header><h1 class="docTitle_3a4h">Pipeline Logstash</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h1><p>These Logstash config files are in /etc/logstash/conf.d/</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-procedures"></a>Logstash Procedures<a class="hash-link" href="#logstash-procedures" title="Direct link to heading">#</a></h2><p>The following steps are defined for logstash:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-inputsconf"></a>01-inputs.conf<a class="hash-link" href="#01-inputsconf" title="Direct link to heading">#</a></h3><p>reads flows from wherever the user wants to get data; in our case, from the netsage_deidentifer_raw rabbit queue.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>drops flows to or from private IP addresses;
 adds @injest_time;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
@@ -36,16 +36,16 @@
 sensor_type    = Circuit, Archive, Exchange Point, or Regional Network
 country_scope = Domestic, International, or Mixed</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-outputsconf--"></a>99-outputs.conf -<a class="hash-link" href="#99-outputsconf--" title="Direct link to heading">#</a></h3><p>adds @exit_time and @processing_time,
 sends results to whereever the user wants them to go. In our case, it sends them to the netsage_archive_input queue (or netsage_ilight_input, etc) on netsage-elk1.grnoc.iu.edu. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In our case, OmniSOC manages the last stage. Their logstash reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-<em> (or om-ns-ilight-</em>, etc).  </p><p>This can be easily replicated with the following configuration though you&#x27;ll need one for each feed/index.</p><p>Naturally the hosts for rabbit and elastic will need to be updated accordingly.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">input {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  rabbitmq {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    host =&gt; &#x27;localhost&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    user =&gt; &#x27;guest&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    password =&gt; &quot;${rabbitmq_pass}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    exchange =&gt; &#x27;netsage.direct&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    key =&gt;   XXXXXXX&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    queue =&gt; &#x27;netsage&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    durable =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    subscription_retry_interval_seconds =&gt; 5</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    connection_timeout =&gt; 10000</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">filter {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  if [@metadata][rabbitmq_properties][timestamp] {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    date {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      match =&gt; [&quot;[@metadata][rabbitmq_properties][timestamp]&quot;, &quot;UNIX&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Once the data is published in elastic, you can use the <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboard</a> to visualize the data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="es-fields"></a>ES fields<a class="hash-link" href="#es-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.</td><td>equivalent to an sql table</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>document id, set by ES</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>type</td><td>flow</td><td>Always &quot;flow&quot; for us. Other types may be &quot;macy&quot;, etc.</td></tr><tr><td>@injest_time</td><td>2020-06-09T21:51:57.059Z</td><td>Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline (99-outputs.conf)</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @injest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>1</td><td>Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Assigned flow id</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>Sflow, Netflow, or Tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Assigned sensor group</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Assigned sensor type</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly"></a>Source Fields (Destination Fields similarly)<a class="hash-link" href="#source-fields-destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>ASN of the IP from geoip ASN database or the ASN from the flow header</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of IP from geoip database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of IP from geoip database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of IP from geoip database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of IP from geoip database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS of the IP from geoip ASN database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly"></a>Source Science Registry Fields  (Destination Fields similarly)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.resource</td><td>Stanford - ImageNet</td><td>Resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td>-</td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.discipline</td><td>CS. Intelligent Systems</td><td>The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Stanford University</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Stanford</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.projects</td><td>.</td><td>Can be an array of projects [we may change this field name soon]</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s location, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.172</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="preferred-fields"></a>&quot;Preferred&quot; fields<a class="hash-link" href="#preferred-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values.</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Otherwise, they are the geoip values.</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="values"></a>Values<a class="hash-link" href="#values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in all the stitched flows</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in all the stitched flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-values"></a>Tstat Values<a class="hash-link" href="#tstat-values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/devel/docker"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-procedures" class="table-of-contents__link">Logstash Procedures</a><ul><li><a href="#01-inputsconf" class="table-of-contents__link">01-inputs.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#50-geoip-taggingconf" class="table-of-contents__link">50-geoip-tagging.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#65-preferred-location-orgconf" class="table-of-contents__link">65-preferred-location-org.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#99-outputsconf--" class="table-of-contents__link">99-outputs.conf -</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a><ul><li><a href="#es-fields" class="table-of-contents__link">ES fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer fields</a></li><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly" class="table-of-contents__link">Source Fields (Destination Fields similarly)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly)</a></li><li><a href="#preferred-fields" class="table-of-contents__link">&quot;Preferred&quot; fields</a></li><li><a href="#values" class="table-of-contents__link">Values</a></li><li><a href="#tstat-values" class="table-of-contents__link">Tstat Values</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/6c279c12.cc8d8222.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/a7459da2.03c88ff1.js"></script>
+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Once the data is published in elastic, you can use the <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboard</a> to visualize the data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="es-fields"></a>ES fields<a class="hash-link" href="#es-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.</td><td>equivalent to an sql table</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>document id, set by ES</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>type</td><td>flow</td><td>Always &quot;flow&quot; for us. Other types may be &quot;macy&quot;, etc.</td></tr><tr><td>@injest_time</td><td>2020-06-09T21:51:57.059Z</td><td>Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline (99-outputs.conf)</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @injest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>1</td><td>Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Assigned flow id</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>Sflow, Netflow, or Tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Assigned sensor group</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Assigned sensor type</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly"></a>Source Fields (Destination Fields similarly)<a class="hash-link" href="#source-fields-destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>ASN of the IP from geoip ASN database or the ASN from the flow header</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of IP from geoip database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of IP from geoip database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of IP from geoip database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of IP from geoip database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS of the IP from geoip ASN database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly"></a>Source Science Registry Fields  (Destination Fields similarly)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.resource</td><td>Stanford - ImageNet</td><td>Resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td>-</td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.discipline</td><td>CS. Intelligent Systems</td><td>The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Stanford University</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Stanford</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.projects</td><td>.</td><td>Can be an array of projects [we may change this field name soon]</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s location, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.172</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="preferred-fields"></a>&quot;Preferred&quot; fields<a class="hash-link" href="#preferred-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values.</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Otherwise, they are the geoip values.</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="values"></a>Values<a class="hash-link" href="#values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in all the stitched flows</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in all the stitched flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-values"></a>Tstat Values<a class="hash-link" href="#tstat-values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.5/pipeline_logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.5/devel/docker"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-procedures" class="table-of-contents__link">Logstash Procedures</a><ul><li><a href="#01-inputsconf" class="table-of-contents__link">01-inputs.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#50-geoip-taggingconf" class="table-of-contents__link">50-geoip-tagging.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#65-preferred-location-orgconf" class="table-of-contents__link">65-preferred-location-org.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#99-outputsconf--" class="table-of-contents__link">99-outputs.conf -</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a><ul><li><a href="#es-fields" class="table-of-contents__link">ES fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer fields</a></li><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly" class="table-of-contents__link">Source Fields (Destination Fields similarly)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly)</a></li><li><a href="#preferred-fields" class="table-of-contents__link">&quot;Preferred&quot; fields</a></li><li><a href="#values" class="table-of-contents__link">Values</a></li><li><a href="#tstat-values" class="table-of-contents__link">Tstat Values</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/6c279c12.225c0b15.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/a7459da2.2d033572.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/deploy/choose_install/index.html b/docs/1.2.6/deploy/choose_install/index.html
index 7dfa15ba..f6a14072 100644
--- a/docs/1.2.6/deploy/choose_install/index.html
+++ b/docs/1.2.6/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing Install | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Choosing Install | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="BareMetal"><meta data-react-helmet="true" property="og:description" content="BareMetal"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5479f42.d126a913.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5479f42.b8876961.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal"></a>BareMetal<a class="hash-link" href="#baremetal" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker</a></li><li><a href="/netsage-pipeline/docs/1.2.6/deploy/install">BareMetal</a></li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/devel/docker"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Install Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal" class="table-of-contents__link">BareMetal</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d5479f42.d126a913.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal"></a>BareMetal<a class="hash-link" href="#baremetal" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker</a></li><li><a href="/netsage-pipeline/docs/1.2.6/deploy/install">BareMetal</a></li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/devel/docker"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Install Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal" class="table-of-contents__link">BareMetal</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d5479f42.b8876961.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/deploy/docker_install/index.html b/docs/1.2.6/deploy/docker_install/index.html
index 08094990..c48eb044 100644
--- a/docs/1.2.6/deploy/docker_install/index.html
+++ b/docs/1.2.6/deploy/docker_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all"><meta data-react-helmet="true" property="og:description" content="The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9feaf8ab.31cb6a73.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9feaf8ab.8c912915.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/docker_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>The docker pattern is provided as much simpler and easier to use pattern that allows you to process and send data without having to deal with all
 the nuances of getting the pipeline setup.</p><p>Before we start, you may have a read over the <a href="/netsage-pipeline/docs/1.2.6/devel/docker">developer docker guide</a> it contains several notes such as how to select the docker version and likely other bits of information you may find useful.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>Nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h2><p>Note that no matter if you use a localized version or take advantage of the docker container already built. You will need to configure your routers to send nfdump stats to the process collecting data on the host:port that you&#x27;ll be defining.</p><p>More info of nfdump can be found <a href="https://github.com/phaag/nfdump/" target="_blank" rel="noopener noreferrer">here</a></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="external-nfdump"></a>External Nfdump<a class="hash-link" href="#external-nfdump" title="Direct link to heading">#</a></h3><p>In this case you have nfdump running in your network somewhere and would like to keep on using it rather then relying on the container provided.</p><p>You&#x27;ll need to update your scripts to output to \$PROJECT/data/input_data. Naturally all the paths are configurable but you&#x27;ll have a much easier if you stick to the defaults.</p><p>If you do choose to store the data elsewhere, the location may still need to be inside of the \$PROJECT or a docker volume location in order for docker to be able to reference it.</p><p>You will also need to configure your routers to point to the nfdump hostname and port in order for nfdump to collect data.</p><p>At this point please proceed to <a href="#common-pattern">Common Pattern</a></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-nfdump"></a>Dockerized Nfdump<a class="hash-link" href="#dockerized-nfdump" title="Direct link to heading">#</a></h3><p>After you&#x27;ve selected the version of docker you&#x27;ll be running. you can start the collector by simply running:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The default version is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p><p>By default the container comes up and will write data to <code>./data/input_data</code> and listen to udp traffic on localhost:9999.</p><p>continue to <a href="#common-pattern">Common Pattern</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="common-pattern"></a>Common Pattern<a class="hash-link" href="#common-pattern" title="Direct link to heading">#</a></h2><p>Before continuing you need to choose if you are going to be use the &#x27;Develop&#x27; version which has the latest changes but might be a bit less stable or using the &#x27;Release&#x27; version.</p><ul><li><p>Development Version</p><ul><li>Disregard anything about <code>docker_select_version.sh</code> that will not apply to you </li><li>Update to latest code git pull origin master</li></ul></li><li><p>Release version</p><ul><li><code>git fetch; git checkout &lt;tag name&gt;</code> replace &quot;tag name&quot; with v1.2.5 or the version you intend to use.</li><li>Please select the version you wish to use using <code>./scripts/docker_select_version.sh</code>
 it is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.</li></ul></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment file<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>Please make a copy of the .env and refer back to the docker <a href="/netsage-pipeline/docs/1.2.6/devel/docker">dev guide</a> on details on configuring the env. Most of the default value should work just fine.</p><p>The only major change you should be aware of are the following values. The output host defines where the final data will land. The sensorName defines what the data will be labeled as.</p><p>If you don&#x27;t send a sensor name it&#x27;ll use the default docker hostname which changes each time you run the pipeline.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=sflowSensorName</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=netflowSensorName</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -35,16 +35,16 @@
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="5"><li>Update the environment file.</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=example</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="6"><li>At this point, please update the router configuration to send data to the new port you&#x27;ve defined.  If the new collector is listening on 0.0.0.0:1234/udp then all traffic you wish grouped under </li></ol><p>the new sensor should be send to 1234/udp.  </p><p>You will need to repeat steps 3-6 for each collector you&#x27;re adding.  For each new configuration the path, sensorName and exposed port have to be unique.  Besides that, there is no limit
 outside of the bounds of the host&#x27;s resources to how many collectors you may run.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="kibana-and-elastic-search"></a>Kibana and Elastic Search<a class="hash-link" href="#kibana-and-elastic-search" title="Direct link to heading">#</a></h3><p>The file docker-compose.develop.yaml can be found in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.6/devel/docker#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  hostname:9999 is the default.</li><li>Make sure you created a .env file and updated the settings accordingly.</li><li>sensorName especially since that identifies the source of the data. </li><li>check the logs of the various components to see if anything jumps out as being invalid.  docker-compose logs -f &lt;service_label&gt;</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory.</p><p>Applying this snippet to logstash may help. Naturally the values will have to change.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>If your only changes are the version you selected simply reset and discard your changes.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development"></a>Development<a class="hash-link" href="#development" title="Direct link to heading">#</a></h4><p>Update the git repo. Likely this won&#x27;t change anything but it&#x27;s always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="release"></a>Release<a class="hash-link" href="#release" title="Direct link to heading">#</a></h4><ol><li>git checkout &lt;tag_value&gt; (ie. v1.2.6, v1.2.7 etc)</li><li><code>./scripts/docker_select_version.sh</code> select the same version as the tag you checked out.</li></ol><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/docker_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Install Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#nfdump" class="table-of-contents__link">Nfdump</a><ul><li><a href="#external-nfdump" class="table-of-contents__link">External Nfdump</a></li><li><a href="#dockerized-nfdump" class="table-of-contents__link">Dockerized Nfdump</a></li></ul></li><li><a href="#common-pattern" class="table-of-contents__link">Common Pattern</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment file</a></li><li><a href="#bringing-up-the-pipeline" class="table-of-contents__link">Bringing up the Pipeline</a></li><li><a href="#shutting-down-the-pipeline" class="table-of-contents__link">Shutting Down the pipeline.</a></li><li><a href="#advanced-configuration" class="table-of-contents__link">Advanced Configuration</a></li><li><a href="#kibana-and-elastic-search" class="table-of-contents__link">Kibana and Elastic Search</a></li></ul></li><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a><ul><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9feaf8ab.31cb6a73.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>If your only changes are the version you selected simply reset and discard your changes.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development"></a>Development<a class="hash-link" href="#development" title="Direct link to heading">#</a></h4><p>Update the git repo. Likely this won&#x27;t change anything but it&#x27;s always a good practice to have the latest version. You will need to do at least a git fetch in order to see the latest tags.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="release"></a>Release<a class="hash-link" href="#release" title="Direct link to heading">#</a></h4><ol><li>git checkout &lt;tag_value&gt; (ie. v1.2.6, v1.2.7 etc)</li><li><code>./scripts/docker_select_version.sh</code> select the same version as the tag you checked out.</li></ol><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/docker_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Install Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#nfdump" class="table-of-contents__link">Nfdump</a><ul><li><a href="#external-nfdump" class="table-of-contents__link">External Nfdump</a></li><li><a href="#dockerized-nfdump" class="table-of-contents__link">Dockerized Nfdump</a></li></ul></li><li><a href="#common-pattern" class="table-of-contents__link">Common Pattern</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment file</a></li><li><a href="#bringing-up-the-pipeline" class="table-of-contents__link">Bringing up the Pipeline</a></li><li><a href="#shutting-down-the-pipeline" class="table-of-contents__link">Shutting Down the pipeline.</a></li><li><a href="#advanced-configuration" class="table-of-contents__link">Advanced Configuration</a></li><li><a href="#kibana-and-elastic-search" class="table-of-contents__link">Kibana and Elastic Search</a></li></ul></li><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a><ul><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9feaf8ab.8c912915.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/deploy/install/index.html b/docs/1.2.6/deploy/install/index.html
index 1d9a45b8..487e88aa 100644
--- a/docs/1.2.6/deploy/install/index.html
+++ b/docs/1.2.6/deploy/install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">NetSage Flow Processing Pipeline Install Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="NetSage Flow Processing Pipeline Install Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e2be698d.6ab0c41b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e2be698d.dd0e2d10.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Install Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>Minimum components</p><ul><li>Data Injestion Source (nfdump, tstat or both)</li><li>RabbitMQ</li><li>LogStash</li><li>Importer (If you use nfdump)</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="prerequirements"></a>Prerequirements<a class="hash-link" href="#prerequirements" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed.</p><ol><li>nfdump </li><li>tstat</li></ol><p>You&#x27;ll need to have at least one of them setup in order to be able to process the data</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h3><p>The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>Once nfdump is setup you&#x27;ll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbitmq"></a>RabbitMQ<a class="hash-link" href="#rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">`</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/deploy/install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Install Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>Minimum components</p><ul><li>Data Injestion Source (nfdump, tstat or both)</li><li>RabbitMQ</li><li>LogStash</li><li>Importer (If you use nfdump)</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="prerequirements"></a>Prerequirements<a class="hash-link" href="#prerequirements" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to injest in order to do anything.  There are two types of data that are consumed.</p><ol><li>nfdump </li><li>tstat</li></ol><p>You&#x27;ll need to have at least one of them setup in order to be able to process the data</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="nfdump"></a>nfdump<a class="hash-link" href="#nfdump" title="Direct link to heading">#</a></h3><p>The NetFlow Importer daemon requires nfdump. If you are only using tstat, you do not need nfdump. nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>Once nfdump is setup you&#x27;ll need to configures your routers to send nfdump to the running process that will save data to a particular location on disk. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbitmq"></a>RabbitMQ<a class="hash-link" href="#rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">`</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, if any. Then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline"></a>Installing the Pipeline<a class="hash-link" href="#installing-the-pipeline" title="Direct link to heading">#</a></h2><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data as we do not want to process all of it. Not needed otherwise.</li><li>Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors, have them</li></ol><p>send directly to the appropriate rabbit queue. </p><ol start="3"><li>Logstash configs - These are executed in alphabetical order. They read events from a rabbit queue, aggregate (ie stitch flows </li></ol><p>that were split between different nfcapd files), add information from geoIP and Science Registry data, and write to a final rabbit queue.
 The final rabbit queue is read by an independent logstash instance and events are put into elasticsearch. One could also modify the
 last logstash conf here to write to elasticsearch.</p><p>Nothing will automatically start after installation as we need to move on to configuration. Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p>Theshared config file, used by all the non-logstash pipeline components, is read before reading the individual config files [THERE USED TO BE MANY DAEMONS INSTEAD OF LOGSTASH. We should redo this]. This allows you to easily configure values that apply to all stages, while allowing you to override them in the individual config files, if desired. A default shared config file is included: <code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code> </p><p>The first, and most important, part of the configuration is the collection(s) this host will import. These are defined as follows:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;!--  &quot;instance&quot; goes along with sensor</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       This is to identify various instances if a sensor has more than one &quot;stream&quot; / data collection</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       Defaults to 0.</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;instance&gt;1&lt;/instance&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">--&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">       Defaults to sensor, but you can set it to something else here</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      &lt;router-address&gt;&lt;/router-address&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    Flow type: type of flow data (defaults to netflow)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &lt;!--</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;/collection&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Notice that <code>instance</code> , <code>router-address</code> , and <code>flow-type</code> are commented out. You only need these if you need an something other than the default values, as described in the comments in the default shared config file.</p><p>You can have multiple <code>collection</code> stanzas, to import multiple collections on one host.</p><p>The shared config looks like this. Note that RabbitMQ connection information is listed, but not the queue or channel, as these will vary per daemon. If you&#x27;re running a default RabbitMQ config, which is open only to &#x27;localhost&#x27; as guest/guest, you won&#x27;t need to change anything here. Note that you will need to change the rabbit_output for the Finished Flow Mover Daemon regardless (see below).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;config&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files1&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname1.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;flow-path&gt;/path/to/flow-files2&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;sensor&gt;hostname2.tld&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -33,16 +33,16 @@
 Create logstash.keystore in /etc/logstash/: (use the same directory as logstash.yml)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ create</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    You can set a password for the keystore itself if you want to investigate that; otherwise skip it.</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_username     (enter username when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_input_pw           (enter password when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_username    (enter username when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore --path.settings /etc/logstash/ add rabbitmq_output_pw          (enter password when prompted)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">```sh</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">To list the keys:</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>  $ sudo -E logstash-keystore list</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">To remove a key-value pair:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">```sh</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  $ sudo -E logstash-keystore remove &lt;key name&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>FLOW STITCHING - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use &quot;-w 1&quot;.</p><p>See the comments in 04-stitching.conf to learn more about how complete flows are defined.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#prerequirements" class="table-of-contents__link">Prerequirements</a><ul><li><a href="#nfdump" class="table-of-contents__link">nfdump</a></li></ul></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#rabbitmq" class="table-of-contents__link">RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline" class="table-of-contents__link">Installing the Pipeline</a></li><li><a href="#configuration" class="table-of-contents__link">Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#configuring-the-pipeline-stages" class="table-of-contents__link">Configuring the Pipeline Stages</a></li><li><a href="#shared-config-file-listing" class="table-of-contents__link">Shared config file listing</a></li></ul></li><li><a href="#running-the-daemons" class="table-of-contents__link">Running the daemons</a><ul><li><a href="#daemon-listing" class="table-of-contents__link">Daemon Listing</a></li></ul></li><li><a href="#setup-notes" class="table-of-contents__link">Setup Notes</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e2be698d.6ab0c41b.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml (default settingss) and/or /etc/logstash/pipelines.yml (settings take precedence). When running logstash on the command line, use &quot;-w 1&quot;.</p><p>See the comments in 04-stitching.conf to learn more about how complete flows are defined.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/deploy/install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#prerequirements" class="table-of-contents__link">Prerequirements</a><ul><li><a href="#nfdump" class="table-of-contents__link">nfdump</a></li></ul></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#rabbitmq" class="table-of-contents__link">RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline" class="table-of-contents__link">Installing the Pipeline</a></li><li><a href="#configuration" class="table-of-contents__link">Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#configuring-the-pipeline-stages" class="table-of-contents__link">Configuring the Pipeline Stages</a></li><li><a href="#shared-config-file-listing" class="table-of-contents__link">Shared config file listing</a></li></ul></li><li><a href="#running-the-daemons" class="table-of-contents__link">Running the daemons</a><ul><li><a href="#daemon-listing" class="table-of-contents__link">Daemon Listing</a></li></ul></li><li><a href="#setup-notes" class="table-of-contents__link">Setup Notes</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e2be698d.dd0e2d10.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/devel/docker/index.html b/docs/1.2.6/devel/docker/index.html
index ace260f4..aec791df 100644
--- a/docs/1.2.6/devel/docker/index.html
+++ b/docs/1.2.6/devel/docker/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Docker Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Docker Setup"><meta data-react-helmet="true" property="og:description" content="Docker Setup"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/aa98ec4a.ffbdd576.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/aa98ec4a.39b5137c.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Docker Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><p><code>scripts/docker_select_version.sh tagValue</code></p><p>Example:</p><p><code>scripts/docker_select_version.sh v1.2.5</code></p><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/devel/docker">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/devel/docker">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Docker Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><p><code>scripts/docker_select_version.sh tagValue</code></p><p>Example:</p><p><code>scripts/docker_select_version.sh v1.2.5</code></p><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
 any git checkout.  Simply follow the directions below and setup your pipeline as instructed.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-base-images"></a>Build Base Images<a class="hash-link" href="#build-base-images" title="Direct link to heading">#</a></h2><p>This is optional.  The image are published on docker hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in this GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuring-the-containers"></a>Configuring the Containers<a class="hash-link" href="#configuring-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t done so already, copy env.example and update it to match your own settings:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbit"></a>Rabbit<a class="hash-link" href="#rabbit" title="Direct link to heading">#</a></h3><p>This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set
 here should be consistent with the config for the logstash and importer </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_ERLANG_COOKIE=&#x27;secret cookie&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBIT_HOST=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_USER=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_PASS=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">discovery.type=single-node</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it&#x27;s set to rabbit</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ <strong>NOTE: Changes will require you to rebuild the container</strong></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h3><p>Define the input rabbit queue.  This should match the importer output queue</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line:  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Logstash</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/aa98ec4a.ffbdd576.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line:  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Logstash</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/aa98ec4a.39b5137c.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/docusaurus/index.html b/docs/1.2.6/docusaurus/index.html
index 6d619fe4..160fe1fb 100644
--- a/docs/1.2.6/docusaurus/index.html
+++ b/docs/1.2.6/docusaurus/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Documentation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Documentation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d91f9def.b4c78c77.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d91f9def.f3256090.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  </p><p>To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation
-at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/DOCUMENTATION.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d91f9def.b4c78c77.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.  </p><p>To extend the doc simply create a markdown file and reference the ID in the side bar.  Please see the related documentation
+at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/DOCUMENTATION.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d91f9def.f3256090.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/pipeline/index.html b/docs/1.2.6/pipeline/index.html
index 560aa29f..6fd745c8 100644
--- a/docs/1.2.6/pipeline/index.html
+++ b/docs/1.2.6/pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ebfaece6.fbef59fb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ebfaece6.d97235a2.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Pipeline</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h1><p>The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>The Pipeline is made of the following components (Currently)</p><ul><li><a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">Importer</a>  (Collection of perl scripts)<ul><li><a href="/netsage-pipeline/docs/1.2.6/pipeline_importer">doc</a></li></ul></li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Elastic Logstash</a> Performs a variety of transformation on the data<ul><li><a href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">doc</a> </li></ul></li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a> used for message passing and queing of tasks.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sensors-and-data-collection"></a>Sensors and Data Collection<a class="hash-link" href="#sensors-and-data-collection" title="Direct link to heading">#</a></h2><p>&quot;Testpoints&quot; or &quot;sensors&quot; collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). </p><p>Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#sensors-and-data-collection" class="table-of-contents__link">Sensors and Data Collection</a><ul><li><a href="#importer" class="table-of-contents__link">Importer</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ebfaece6.fbef59fb.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Pipeline</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h1><p>The Netsage Flow Processing Pipeline includes several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="components"></a>Components<a class="hash-link" href="#components" title="Direct link to heading">#</a></h2><p>The Pipeline is made of the following components (Currently)</p><ul><li><a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">Importer</a>  (Collection of perl scripts)<ul><li><a href="/netsage-pipeline/docs/1.2.6/pipeline_importer">doc</a></li></ul></li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Elastic Logstash</a> Performs a variety of transformation on the data<ul><li><a href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">doc</a> </li></ul></li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a> used for message passing and queing of tasks.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sensors-and-data-collection"></a>Sensors and Data Collection<a class="hash-link" href="#sensors-and-data-collection" title="Direct link to heading">#</a></h2><p>&quot;Testpoints&quot; or &quot;sensors&quot; collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing (for globanoc, flow-proc.bldc.grnoc.iu.edu or netsage-probe1.grnoc.iu.edu). </p><p>Tstat data goes directly into the netsage_deidentifier_raw queue rabbit queue. The other data is written to nfcapd files.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#components" class="table-of-contents__link">Components</a></li><li><a href="#sensors-and-data-collection" class="table-of-contents__link">Sensors and Data Collection</a><ul><li><a href="#importer" class="table-of-contents__link">Importer</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ebfaece6.d97235a2.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/pipeline_importer/index.html b/docs/1.2.6/pipeline_importer/index.html
index 2447d7d4..13ec31af 100644
--- a/docs/1.2.6/pipeline_importer/index.html
+++ b/docs/1.2.6/pipeline_importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Importer"><meta data-react-helmet="true" property="og:description" content="Importer"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/21ee0bc4.17500718.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/21ee0bc4.03721b9e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-1"></a>Importer<a class="hash-link" href="#importer-1" title="Direct link to heading">#</a></h2><p>NOTE: Importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>configuration files for the importer are <code>netsage_netflow_importer.xml</code> and <code>netsage_shared.xml</code> in <code>/etc/grnoc/netsage/deidentifer/</code>. Comments in the files briefly describe the options.</p><p>Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="internals"></a>Internals<a class="hash-link" href="#internals" title="Direct link to heading">#</a></h3><p>The importer uses the nfdump command with -a to aggregate within the file, and -L <code>threshold</code> to throw out any flows under a flow size threshold. </p><p>For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.
-A ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages.</p><p>All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Logstash »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#importer-1" class="table-of-contents__link">Importer</a><ul><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li><li><a href="#internals" class="table-of-contents__link">Internals</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/21ee0bc4.17500718.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline_importer">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>A netsage-netflow-importer-daemon reads any new nfcapd files that have come in after a configurable delay. The importer aggregates flows within each file, and writes the results to the netsage_deidentifier_raw queue rabbit queue.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-1"></a>Importer<a class="hash-link" href="#importer-1" title="Direct link to heading">#</a></h2><p>NOTE: Importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>configuration files for the importer are <code>netsage_netflow_importer.xml</code> and <code>netsage_shared.xml</code> in <code>/etc/grnoc/netsage/deidentifer/</code>. Comments in the files briefly describe the options.</p><p>Names of files have already been read are stored in /var/cache/netsage/netflow_importer.cache. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="internals"></a>Internals<a class="hash-link" href="#internals" title="Direct link to heading">#</a></h3><p>The importer uses the nfdump command with -a to aggregate within the file, and -L <code>threshold</code> to throw out any flows under a flow size threshold. </p><p>For cenic, data from the importer first goes into a ...prefilter queue. A netsage-flow-filter-daemon reads it out, removes some flows , then sends it to the ...raw queue.
+A ...raw2 or ...fake queue is created for historical reasons but never actually holds any messages.</p><p>All flow data waits in the netsage_deidentifier_raw queue until it is processed by the logstash pipeline as follows.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Logstash »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#importer-1" class="table-of-contents__link">Importer</a><ul><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li><li><a href="#internals" class="table-of-contents__link">Internals</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/21ee0bc4.03721b9e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.6/pipeline_logstash/index.html b/docs/1.2.6/pipeline_logstash/index.html
index b547e162..63cfc905 100644
--- a/docs/1.2.6/pipeline_logstash/index.html
+++ b/docs/1.2.6/pipeline_logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Logstash | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.6"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.6"><meta data-react-helmet="true" property="og:title" content="Pipeline Logstash | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Logstash"><meta data-react-helmet="true" property="og:description" content="Logstash"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/38d1fe39.6ca6725c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/38d1fe39.e28e353a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Pipeline Logstash</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h1><p>These Logstash config files are in /etc/logstash/conf.d/</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The following steps are defined for logstash:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (&quot;.disabled&quot; can be removed from other input configs to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline_logstash">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.6/pipeline_logstash">Logstash</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Developer Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/devel/docker">Docker Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/install">Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/deploy/docker_install">Docker Install</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.6/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.6</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.6</span></div><header><h1 class="docTitle_3a4h">Pipeline Logstash</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h1><p>These Logstash config files are in /etc/logstash/conf.d/</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The following steps are defined for logstash:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (&quot;.disabled&quot; can be removed from other input configs to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 adds @ingest_time (this is mainly for developers);
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
@@ -33,16 +33,16 @@
 </em>sensor_type    = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)
 <em>country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)
 </em>is_network_testing = yes, no  (yes if discipline = CS.Network Testing and Monitoring or port = 5001, 5101, or 5201)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a rabbitmq queue (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In our case, OmniSOC manages the last stage. Their logstash instance reads flows from the netsage_archive_input queue and sends it into elasticsearch. The indices are named like om-ns-netsage-YYYY.mm.dd-<em> (or om-ns-ilight-</em>, etc).  </p><p>This can be easily replicated with the following configuration though you&#x27;ll need one for each feed/index.</p><p>Naturally the hosts for rabbit and elastic will need to be updated accordingly.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">input {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  rabbitmq {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    host =&gt; &#x27;localhost&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    user =&gt; &#x27;guest&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    password =&gt; &quot;${rabbitmq_pass}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    exchange =&gt; &#x27;netsage.direct&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    key =&gt;   XXXXXXX&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    queue =&gt; &#x27;netsage&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    durable =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    subscription_retry_interval_seconds =&gt; 5</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    connection_timeout =&gt; 10000</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">filter {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  if [@metadata][rabbitmq_properties][timestamp] {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    date {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      match =&gt; [&quot;[@metadata][rabbitmq_properties][timestamp]&quot;, &quot;UNIX&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Once the data is published in elastic, you can use the <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboard</a> to visualize the data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="es-fields"></a>ES fields<a class="hash-link" href="#es-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.</td><td>equivalent to an sql table</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>document id, set by ES</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>type</td><td>flow</td><td>Always &quot;flow&quot; for us. Other types may be &quot;macy&quot;, etc.</td></tr><tr><td>@injest_time</td><td>2020-06-09T21:51:57.059Z</td><td>Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline (99-outputs.conf)</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @injest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>1</td><td>Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Assigned flow id</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>Sflow, Netflow, or Tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Assigned sensor group</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Assigned sensor type</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly"></a>Source Fields (Destination Fields similarly)<a class="hash-link" href="#source-fields-destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>ASN of the IP from geoip ASN database or the ASN from the flow header</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of IP from geoip database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of IP from geoip database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of IP from geoip database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of IP from geoip database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS of the IP from geoip ASN database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly"></a>Source Science Registry Fields  (Destination Fields similarly)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.resource</td><td>Stanford - ImageNet</td><td>Resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td>-</td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.discipline</td><td>CS. Intelligent Systems</td><td>The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Stanford University</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Stanford</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.projects</td><td>.</td><td>Can be an array of projects [we may change this field name soon]</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s location, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.172</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="preferred-fields"></a>&quot;Preferred&quot; fields<a class="hash-link" href="#preferred-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values.</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Otherwise, they are the geoip values.</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="values"></a>Values<a class="hash-link" href="#values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in all the stitched flows</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in all the stitched flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-values"></a>Tstat Values<a class="hash-link" href="#tstat-values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/devel/docker"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a><ul><li><a href="#es-fields" class="table-of-contents__link">ES fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer fields</a></li><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly" class="table-of-contents__link">Source Fields (Destination Fields similarly)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly)</a></li><li><a href="#preferred-fields" class="table-of-contents__link">&quot;Preferred&quot; fields</a></li><li><a href="#values" class="table-of-contents__link">Values</a></li><li><a href="#tstat-values" class="table-of-contents__link">Tstat Values</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/7d380c4c.ad3c0101.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/38d1fe39.6ca6725c.js"></script>
+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Once the data is published in elastic, you can use the <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboard</a> to visualize the data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="es-fields"></a>ES fields<a class="hash-link" href="#es-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.</td><td>equivalent to an sql table</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>document id, set by ES</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>type</td><td>flow</td><td>Always &quot;flow&quot; for us. Other types may be &quot;macy&quot;, etc.</td></tr><tr><td>@injest_time</td><td>2020-06-09T21:51:57.059Z</td><td>Essentially time the flow went into the logstash pipeline (10-preliminaries.conf for tstat flows) or the time stitching of the flow commenced (40-aggregation.conf for others)</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline (99-outputs.conf)</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @injest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>1</td><td>Number of flows stitched together to make this final one. 0 for tstat flows, which are always complete. 1 if no flows were stitched together.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Assigned flow id</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>Sflow, Netflow, or Tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Assigned sensor group</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Assigned sensor type</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly"></a>Source Fields (Destination Fields similarly)<a class="hash-link" href="#source-fields-destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>ASN of the IP from geoip ASN database or the ASN from the flow header</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of IP from geoip database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of IP from geoip database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of IP from geoip database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of IP from geoip database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS of the IP from geoip ASN database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly"></a>Source Science Registry Fields  (Destination Fields similarly)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.resource</td><td>Stanford - ImageNet</td><td>Resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td>-</td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.discipline</td><td>CS. Intelligent Systems</td><td>The science discipline that uses the resource (ie host). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Stanford University</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Stanford</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.projects</td><td>.</td><td>Can be an array of projects [we may change this field name soon]</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s location, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.172</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="preferred-fields"></a>&quot;Preferred&quot; fields<a class="hash-link" href="#preferred-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values.</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Otherwise, they are the geoip values.</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="values"></a>Values<a class="hash-link" href="#values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in all the stitched flows</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in all the stitched flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-values"></a>Tstat Values<a class="hash-link" href="#tstat-values" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.6/pipeline_logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/pipeline_importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.6/devel/docker"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a><ul><li><a href="#es-fields" class="table-of-contents__link">ES fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer fields</a></li><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly" class="table-of-contents__link">Source Fields (Destination Fields similarly)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly)</a></li><li><a href="#preferred-fields" class="table-of-contents__link">&quot;Preferred&quot; fields</a></li><li><a href="#values" class="table-of-contents__link">Values</a></li><li><a href="#tstat-values" class="table-of-contents__link">Tstat Values</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/7d380c4c.b92baa88.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/38d1fe39.e28e353a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/components/docker_env/index.html b/docs/1.2.7/components/docker_env/index.html
index 24038dff..c79b6ecf 100644
--- a/docs/1.2.7/components/docker_env/index.html
+++ b/docs/1.2.7/components/docker_env/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Please copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Please copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/358e6759.d9fbc6e9.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/358e6759.19972cb0.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=&quot;my sflow sensor name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=&quot;my netflow sensor name&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers and you&#x27;re good to go. (Use quotes if the names have spaces.) </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and can be used in visualizations. Choose names that are meaningful and unique.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=&quot;my sflow sensor name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=&quot;my netflow sensor name&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers and you&#x27;re good to go. (Use quotes if the names have spaces.) </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and can be used in visualizations. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;RNDNet Sflow&quot; and &quot;RNDNet Netflow&quot; or &quot;rtr.one.rndnet.edu&quot; and &quot;rtr.two.nrdnet.edu&quot;. Whatever makes sense in your situation.</p></div></div><ul><li>If you don&#x27;t set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used.</li><li>If you have only one collector, comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li><li>If you&#x27;re not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don&#x27;t start up either collector.</li></ul><p>Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware.</p><p><strong>rabbit_output_host</strong>: this defines where the final data will land after going through the pipeline.  Use the default <code>rabbit</code> for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name).</p><p>The Logstash Aggregation Filter settings are exposed in case you wish to use different values.
 (See comments in the *-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
-on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/358e6759.d9fbc6e9.js"></script>
+on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/358e6759.19972cb0.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/components/docker_pipeline/index.html b/docs/1.2.7/components/docker_pipeline/index.html
index 1b79d167..180af9be 100644
--- a/docs/1.2.7/components/docker_pipeline/index.html
+++ b/docs/1.2.7/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (importers and logstash) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (importers and logstash) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e436ce66.ffd97d6e.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e436ce66.54f16243.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (importers and logstash) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You can check the logs for each of the container by running</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Shut down the pipeline using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e436ce66.ffd97d6e.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (importers and logstash) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You can check the logs for each of the container by running</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Shut down the pipeline using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e436ce66.54f16243.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/components/docker_upgrade/index.html b/docs/1.2.7/components/docker_upgrade/index.html
index d9f1c581..bd7cbc2f 100644
--- a/docs/1.2.7/components/docker_upgrade/index.html
+++ b/docs/1.2.7/components/docker_upgrade/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_upgrade | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="docker_upgrade | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Update Source Code"><meta data-react-helmet="true" property="og:description" content="Update Source Code"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/4ff24bf8.5f1f44d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/4ff24bf8.c0099967.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-and-collectors"></a>Docker and Collectors<a class="hash-link" href="#docker-and-collectors" title="Direct link to heading">#</a></h3><p>Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, <code>image: netsage/nfdump-collector:1.6.18</code>). Also check the docker version (eg, <code>version: &quot;3.7&quot;</code>) to see if you&#x27;ll need to ugrade docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag_value&quot; </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#docker-and-collectors" class="table-of-contents__link">Docker and Collectors</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/4ff24bf8.5f1f44d6.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-and-collectors"></a>Docker and Collectors<a class="hash-link" href="#docker-and-collectors" title="Direct link to heading">#</a></h3><p>Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, <code>image: netsage/nfdump-collector:1.6.18</code>). Also check the docker version (eg, <code>version: &quot;3.7&quot;</code>) to see if you&#x27;ll need to ugrade docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag_value&quot; </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update docker containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#docker-and-collectors" class="table-of-contents__link">Docker and Collectors</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update docker containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/4ff24bf8.c0099967.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/deploy/bare_metal_install/index.html b/docs/1.2.7/deploy/bare_metal_install/index.html
index 0eda1a5a..2d88d415 100644
--- a/docs/1.2.7/deploy/bare_metal_install/index.html
+++ b/docs/1.2.7/deploy/bare_metal_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/aafd28b2.21d293f7.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/aafd28b2.fb1f547f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash"></a>Installing the Pipeline (Importer and Logstash)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers is set to 1 (unless you have removed the aggregation logstash conf).</li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Sensor name - can be the hostname or any string you like --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;sensor&gt;Netflow Sensor 1d&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Flow type - sflow or netflow (defaults to netflow) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-type&gt;netflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -39,16 +39,16 @@
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash daily. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/aafd28b2.21d293f7.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash daily. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/aafd28b2.fb1f547f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/deploy/choose_install/index.html b/docs/1.2.7/deploy/choose_install/index.html
index 570e5ddc..51294e52 100644
--- a/docs/1.2.7/deploy/choose_install/index.html
+++ b/docs/1.2.7/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing Install | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Choosing Install | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="BareMetal or Server Install"><meta data-react-helmet="true" property="og:description" content="BareMetal or Server Install"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/5c4beb71.a62d61d9.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/5c4beb71.b666c6b3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/5c4beb71.a62d61d9.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/5c4beb71.b666c6b3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/deploy/docker_install_advanced/index.html b/docs/1.2.7/deploy/docker_install_advanced/index.html
index a3c1d5ff..e915f7a2 100644
--- a/docs/1.2.7/deploy/docker_install_advanced/index.html
+++ b/docs/1.2.7/deploy/docker_install_advanced/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><meta data-react-helmet="true" property="og:description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ac3b7d69.fd3bbc4c.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ac3b7d69.6126d7a7.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out while waiting for flows from sensors B-D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-docker-composeoverrideyml"></a>1. docker-compose.override.yml<a class="hash-link" href="#1-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out while waiting for flows from sensors B-D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-docker-composeoverrideyml"></a>1. docker-compose.override.yml<a class="hash-link" href="#1-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
 router should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change the last part of the path to something meaningful.</li></ul><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2--netsage_overridexml"></a>2.  netsage_override.xml<a class="hash-link" href="#2--netsage_overridexml" title="Direct link to heading">#</a></h3><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; it will be replaced with a value set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-environment-file"></a>3. Environment file<a class="hash-link" href="#3-environment-file" title="Direct link to heading">#</a></h3><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=&quot;Example New York sFlow&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-which-version-to-run"></a>Select which version to run<a class="hash-link" href="#select-which-version-to-run" title="Direct link to heading">#</a></h3><p>Before continuing you need to choose if you are going to be use the &#x27;Develop&#x27; version which has the latest changes but might be a bit less stable (or have code not ready for use) or a &#x27;Released&#x27; version. If you&#x27;re opting to use the Develop version, then simply skip the version selection step.</p><div class="admonition admonition-caution alert alert--warning"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>caution</h5></div><div class="admonition-content"><p>Warning! it is HIGHLY recommended to not use the :latest as that is intended to be a developer release. You may still use it but be aware that you may have some instability each time you update.</p></div></div><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  git checkout &quot;tag name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  ./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;tag name&quot; with the version you intend to use, eg, &quot;v1.2.5&quot;.  When prompted by the script, select the same version.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-new-collector"></a>Running the new collector<a class="hash-link" href="#running-the-new-collector" title="Direct link to heading">#</a></h3><p>After selecting the docker version to run, you can start the new collector by running the following line, using the collector name:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="for-tstat-data"></a>For Tstat Data<a class="hash-link" href="#for-tstat-data" title="Direct link to heading">#</a></h2><p>Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline&#x27;s ingest rabbit queue (named &quot;netsage_deidentifier_raw&quot;).  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.</p><p>Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See <a href="https://github.com/netsage-project/tstat-transport.git" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/tstat-transport.git</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-logstash-java-settings"></a>To Customize Logstash Java Settings<a class="hash-link" href="#to-customize-logstash-java-settings" title="Direct link to heading">#</a></h2><p>If you need to modify the amount of memory logstash can use or any other java settings,
 rename the provided example for JVM Options and tweak the settings as desired.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-export-data-to-an-nfs-volume"></a>To Export Data to an NFS Volume<a class="hash-link" href="#to-export-data-to-an-nfs-volume" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either</p><ol><li>export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)</li><li>update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml</li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure to update all the paths correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-docker-composeoverrideyml" class="table-of-contents__link">1. docker-compose.override.yml</a></li><li><a href="#2--netsage_overridexml" class="table-of-contents__link">2.  netsage_override.xml</a></li><li><a href="#3-environment-file" class="table-of-contents__link">3. Environment file</a></li><li><a href="#select-which-version-to-run" class="table-of-contents__link">Select which version to run</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#for-tstat-data" class="table-of-contents__link">For Tstat Data</a></li><li><a href="#to-customize-logstash-java-settings" class="table-of-contents__link">To Customize Logstash Java Settings</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#to-export-data-to-an-nfs-volume" class="table-of-contents__link">To Export Data to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ac3b7d69.fd3bbc4c.js"></script>
+You&#x27;ll have to manage the volumes exported and ensure to update all the paths correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-docker-composeoverrideyml" class="table-of-contents__link">1. docker-compose.override.yml</a></li><li><a href="#2--netsage_overridexml" class="table-of-contents__link">2.  netsage_override.xml</a></li><li><a href="#3-environment-file" class="table-of-contents__link">3. Environment file</a></li><li><a href="#select-which-version-to-run" class="table-of-contents__link">Select which version to run</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#for-tstat-data" class="table-of-contents__link">For Tstat Data</a></li><li><a href="#to-customize-logstash-java-settings" class="table-of-contents__link">To Customize Logstash Java Settings</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#to-export-data-to-an-nfs-volume" class="table-of-contents__link">To Export Data to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ac3b7d69.6126d7a7.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/deploy/docker_install_simple/index.html b/docs/1.2.7/deploy/docker_install_simple/index.html
index 0395d280..a4620707 100644
--- a/docs/1.2.7/deploy/docker_install_simple/index.html
+++ b/docs/1.2.7/deploy/docker_install_simple/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Default Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Docker Default Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see https://github.com/netsage-project/docker-nfdump-collector) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read &quot;Docker Advanced&quot; after reading through this guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see https://github.com/netsage-project/docker-nfdump-collector) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read &quot;Docker Advanced&quot; after reading through this guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/879923b2.2bf3c135.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/879923b2.90f3d9b4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/docker-nfdump-collector</a>) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read &quot;Docker Advanced&quot; after reading through this guide.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="first"></a>First<a class="hash-link" href="#first" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t already, install Docker/compose and clone this project from github (<a href="https://github.com/netsage-project/netsage-pipeline.git" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/netsage-pipeline.git</a>).</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-composeoverrideyml"></a>Docker-compose.override.yml<a class="hash-link" href="#docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern for running the Pipeline, with docker-based collectors, is defined in the docker-compose.override_example.yml. Copy this to docker-compose.override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default this will bring up a single netflow collector and a single sflow collector. For most people this is more than enough. If you&#x27;re sticking to the default you don&#x27;t need to make any changes to the docker-compose.override_example.yml</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose</p></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you are only interested in netflow OR sflow data, you should remove the section for the collector that is not used.</p></div></div><p>This file also specifies port numbers and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to <code>/data/input_data/</code> . Each collector is namespaced by its type so the sflow collector will write data to <code>/data/input_data/sflow/</code> and the netflow collector will write data to <code>/data/input_data/netflow/</code>.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p></p><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=&quot;my sflow sensor name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=&quot;my netflow sensor name&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers and you&#x27;re good to go. (Use quotes if the names have spaces.) </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and can be used in visualizations. Choose names that are meaningful and unique.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy docker-based sflow and netflow collectors (see <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/docker-nfdump-collector</a>) and a basic docker flow processing pipeline. The collectors will save incoming flow data to disk, while the pipeline Importer will read it and pipeline Lostash filters will process it. Without any modification, 1 sflow collector, 1 netflow collector, and a flow processing pipeline will run. If you have only 1 collector, this guide will show you how to disable the unnecessary one.  If you need 2 or more collectors of the same type, please read &quot;Docker Advanced&quot; after reading through this guide.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="first"></a>First<a class="hash-link" href="#first" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t already, install Docker/compose and clone this project from github (<a href="https://github.com/netsage-project/netsage-pipeline.git" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/netsage-pipeline.git</a>).</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-composeoverrideyml"></a>Docker-compose.override.yml<a class="hash-link" href="#docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern for running the Pipeline, with docker-based collectors, is defined in the docker-compose.override_example.yml. Copy this to docker-compose.override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default this will bring up a single netflow collector and a single sflow collector. For most people this is more than enough. If you&#x27;re sticking to the default you don&#x27;t need to make any changes to the docker-compose.override_example.yml</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose</p></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you are only interested in netflow OR sflow data, you should remove the section for the collector that is not used.</p></div></div><p>This file also specifies port numbers and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to <code>/data/input_data/</code> . Each collector is namespaced by its type so the sflow collector will write data to <code>/data/input_data/sflow/</code> and the netflow collector will write data to <code>/data/input_data/netflow/</code>.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p></p><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=&quot;my sflow sensor name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=&quot;my netflow sensor name&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers and you&#x27;re good to go. (Use quotes if the names have spaces.) </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and can be used in visualizations. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;RNDNet Sflow&quot; and &quot;RNDNet Netflow&quot; or &quot;rtr.one.rndnet.edu&quot; and &quot;rtr.two.nrdnet.edu&quot;. Whatever makes sense in your situation.</p></div></div><ul><li>If you don&#x27;t set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used.</li><li>If you have only one collector, comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li><li>If you&#x27;re not using a netflow or an sflow collector (you are getting only tstat data), then simply disregard the env settings and don&#x27;t start up either collector.</li></ul><p>Other settings of note in this file includes the following. You will not necessarily need to change these, but be aware.</p><p><strong>rabbit_output_host</strong>: this defines where the final data will land after going through the pipeline.  Use the default <code>rabbit</code> for the local rabbitMQ server, running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name).</p><p>The Logstash Aggregation Filter settings are exposed in case you wish to use different values.
 (See comments in the *-aggregation.conf file.) This config stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
-on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p><p></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-version"></a>Pipeline Version<a class="hash-link" href="#pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml and finished adjusting it for any customizations, you&#x27;re ready to select your version.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git fetch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;tag name&quot; with the version you intend to use, e.g., &quot;v1.2.5&quot;. Select the same version when prompted by docker_select_version.sh.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h3><p>After selecting the version to run, you can start the two flow collectors by running the following line. If you only need one of the collectors, remove the other from this command.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-pipeline"></a>Running the Pipeline<a class="hash-link" href="#running-the-pipeline" title="Direct link to heading">#</a></h3><p></p><p>Start up the pipeline (importers and logstash) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You can check the logs for each of the container by running</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Shut down the pipeline using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data. </p><p>Sflow and netflow data should be exported to the pipeline host where nfcapd and/or sfcapd collectors are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data. (See the Docker Advanced guide.)</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><p></p><h3>Update Source Code</h3><p>To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Docker and Collectors</h3><p>Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, <code>image: netsage/nfdump-collector:1.6.18</code>). Also check the docker version (eg, <code>version: &quot;3.7&quot;</code>) to see if you&#x27;ll need to ugrade docker.</p><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag_value&quot; </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Update docker containers</h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#first" class="table-of-contents__link">First</a></li><li><a href="#docker-composeoverrideyml" class="table-of-contents__link">Docker-compose.override.yml</a></li><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#pipeline-version" class="table-of-contents__link">Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a></li><li><a href="#running-the-pipeline" class="table-of-contents__link">Running the Pipeline</a></li><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/879923b2.2bf3c135.js"></script>
+on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p><p></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-version"></a>Pipeline Version<a class="hash-link" href="#pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml and finished adjusting it for any customizations, you&#x27;re ready to select your version.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git fetch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag name&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;tag name&quot; with the version you intend to use, e.g., &quot;v1.2.5&quot;. Select the same version when prompted by docker_select_version.sh.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h3><p>After selecting the version to run, you can start the two flow collectors by running the following line. If you only need one of the collectors, remove the other from this command.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-pipeline"></a>Running the Pipeline<a class="hash-link" href="#running-the-pipeline" title="Direct link to heading">#</a></h3><p></p><p>Start up the pipeline (importers and logstash) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You can check the logs for each of the container by running</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Shut down the pipeline using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data. </p><p>Sflow and netflow data should be exported to the pipeline host where nfcapd and/or sfcapd collectors are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data. (See the Docker Advanced guide.)</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><p></p><h3>Update Source Code</h3><p>To do a Pipeline upgrade, just reset and pull changes, including the new release, from github. Your non-example env and override files will not be overwritten, but check the new example files to see if there are any updates to copy in.  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Docker and Collectors</h3><p>Since the collectors live outside of version control, please check the docker-compose.override_example.yml to see if nfdump needs to be updated (eg, <code>image: netsage/nfdump-collector:1.6.18</code>). Also check the docker version (eg, <code>version: &quot;3.7&quot;</code>) to see if you&#x27;ll need to ugrade docker.</p><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout &quot;tag_value&quot; </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Update docker containers</h3><p>This applies for both development and release</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#first" class="table-of-contents__link">First</a></li><li><a href="#docker-composeoverrideyml" class="table-of-contents__link">Docker-compose.override.yml</a></li><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#pipeline-version" class="table-of-contents__link">Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a></li><li><a href="#running-the-pipeline" class="table-of-contents__link">Running the Pipeline</a></li><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/879923b2.90f3d9b4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/deploy/docker_troubleshoot/index.html b/docs/1.2.7/deploy/docker_troubleshoot/index.html
index 16daf06f..69d92085 100644
--- a/docs/1.2.7/deploy/docker_troubleshoot/index.html
+++ b/docs/1.2.7/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d71ff009.7f914194.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d71ff009.037347f4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  hostname:9999 is the default.</li><li>Make sure you created a .env file and updated the settings accordingly.</li><li>sensorName especially since that identifies the source of the data.</li><li>check the logs of the various components to see if anything jumps out as being invalid.  docker-compose logs -f &lt;service_label&gt;</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash which is usually only allocated 1GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory.</p><p>Please see the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced#customize-logstash-settings">advaned section</a> for details on how to customize logstash</p><p>Applying this snippet to logstash may help. Naturally the values will have to change.</p><p>You may also try the pattern below, if that still isn&#x27;t enough.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d71ff009.7f914194.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  hostname:9999 is the default.</li><li>Make sure you created a .env file and updated the settings accordingly.</li><li>sensorName especially since that identifies the source of the data.</li><li>check the logs of the various components to see if anything jumps out as being invalid.  docker-compose logs -f &lt;service_label&gt;</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash which is usually only allocated 1GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory.</p><p>Please see the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced#customize-logstash-settings">advaned section</a> for details on how to customize logstash</p><p>Applying this snippet to logstash may help. Naturally the values will have to change.</p><p>You may also try the pattern below, if that still isn&#x27;t enough.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d71ff009.037347f4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/devel/dev_dataset/index.html b/docs/1.2.7/devel/dev_dataset/index.html
index e6c89dcc..66d6d5f1 100644
--- a/docs/1.2.7/devel/dev_dataset/index.html
+++ b/docs/1.2.7/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/3914abfa.09b700f7.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/3914abfa.df468ae2.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> the best
-test is to replay network data and inspecting the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been annonymized and should have no identifying information. </p><p>You can download the files locally from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note on which port the collectors are listing on.  You&#x27;ll be running the collectors by using a custom docker-compose.override.yml but if you&#x27;re using the example provided as baseline the ports should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the ports are:</p><ul><li>9999/udp for netflow.</li><li>9998/udp for sflow and </li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data use the following command for netflow/sflow respectively</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9998 -r sflow_anon.nfcapd -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/3914abfa.09b700f7.js"></script>
+test is to replay network data and inspecting the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been annonymized and should have no identifying information. </p><p>You can download the files locally from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note on which port the collectors are listing on.  You&#x27;ll be running the collectors by using a custom docker-compose.override.yml but if you&#x27;re using the example provided as baseline the ports should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the ports are:</p><ul><li>9999/udp for netflow.</li><li>9998/udp for sflow and </li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data use the following command for netflow/sflow respectively</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9998 -r sflow_anon.nfcapd -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/3914abfa.df468ae2.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/devel/docker_dev_guide/index.html b/docs/1.2.7/devel/docker_dev_guide/index.html
index 6d8f062b..ccd87347 100644
--- a/docs/1.2.7/devel/docker_dev_guide/index.html
+++ b/docs/1.2.7/devel/docker_dev_guide/index.html
@@ -6,34 +6,34 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Docker Setup"><meta data-react-helmet="true" property="og:description" content="Docker Setup"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/97edd56e.3b71c751.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/97edd56e.d5ab66b5.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
 any git checkout.  Simply follow the directions below and setup your pipeline as instructed.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-base-images"></a>Build Base Images<a class="hash-link" href="#build-base-images" title="Direct link to heading">#</a></h2><p>This is optional.  The image are published on docker hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in this GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuring-the-containers"></a>Configuring the Containers<a class="hash-link" href="#configuring-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t done so already, copy env.example and update it to match your own settings:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbit"></a>Rabbit<a class="hash-link" href="#rabbit" title="Direct link to heading">#</a></h3><p>This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set
 here should be consistent with the config for the logstash and importer </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_ERLANG_COOKIE=&#x27;secret cookie&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBIT_HOST=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_USER=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_PASS=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">discovery.type=single-node</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it&#x27;s set to rabbit</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ <strong>NOTE: Changes will require you to rebuild the container</strong></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h3><p>Define the input rabbit queue.  This should match the importer output queue</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/97edd56e.3b71c751.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/97edd56e.d5ab66b5.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/docusaurus/index.html b/docs/1.2.7/docusaurus/index.html
index 21dcb87f..91e902d4 100644
--- a/docs/1.2.7/docusaurus/index.html
+++ b/docs/1.2.7/docusaurus/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Documentation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Documentation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Using Local Tools"><meta data-react-helmet="true" property="og:description" content="Using Local Tools"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0381afdd.426b41ab.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0381afdd.de4385a8.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
-at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0381afdd.426b41ab.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
+at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0381afdd.de4385a8.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/elastic/index.html b/docs/1.2.7/pipeline/elastic/index.html
index 70780df0..15980cbd 100644
--- a/docs/1.2.7/pipeline/elastic/index.html
+++ b/docs/1.2.7/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/3309db24.62c09093.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/3309db24.f6153687.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>sflow, netflow, or tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (Circuit, Regional Network, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_namess</td><td>ATLAS</td><td>A &quot;Project&quot; that the resource is part</td></tr><tr><td>meta.scireg.src.projects</td><td>...</td><td>An object containing project info</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values, otherwise they are the CAIDA or MaxMind GeoIP values..</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td></td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/3309db24.62c09093.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>sflow, netflow, or tstat</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (Circuit, Regional Network, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>Domestic, International, or Mixed, depending on countries of src and dst</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_namess</td><td>ATLAS</td><td>A &quot;Project&quot; that the resource is part</td></tr><tr><td>meta.scireg.src.projects</td><td>...</td><td>An object containing project info</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, these are the SciReg values, otherwise they are the CAIDA or MaxMind GeoIP values..</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td></td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000</td><td></td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/3309db24.f6153687.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/importer/index.html b/docs/1.2.7/pipeline/importer/index.html
index cd7b7ddc..5bddbbe5 100644
--- a/docs/1.2.7/pipeline/importer/index.html
+++ b/docs/1.2.7/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/43862f47.792b1adb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/43862f47.7a6ac57b.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/43862f47.792b1adb.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/43862f47.7a6ac57b.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/intro/index.html b/docs/1.2.7/pipeline/intro/index.html
index 8e76c793..e8db2dda 100644
--- a/docs/1.2.7/pipeline/intro/index.html
+++ b/docs/1.2.7/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/07faddd5.24f5303f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/07faddd5.b2854b31.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers can be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>[Importer]:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   <a href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Doc</a>. <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">In github</a>.</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a>: Performs a variety of operations on the flow data to transform it and add additional information.  <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Doc</a>.</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow or netflow sensor, or one of each (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/07faddd5.24f5303f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers can be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>[Importer]:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   <a href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Doc</a>. <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">In github</a>.</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a>: Performs a variety of operations on the flow data to transform it and add additional information.  <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Doc</a>.</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow or netflow sensor, or one of each (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/07faddd5.b2854b31.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/logstash/index.html b/docs/1.2.7/pipeline/logstash/index.html
index 788358fa..c0b6179f 100644
--- a/docs/1.2.7/pipeline/logstash/index.html
+++ b/docs/1.2.7/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/bb237b1e.9e57bcc8.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/bb237b1e.ea30ebd1.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf are not saved in github. You will need to provide these yourself or ask us for any we may have compliled. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf are not saved in github. You will need to provide these yourself or ask us for any we may have compliled. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 does some data type conversions;
@@ -29,16 +29,16 @@
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
 If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In GlobalNOC/Netsage Project&#x27;s case, the output filter writes the flows to a RabbitMQ queue on another host and the last stage is a separate logstash pipeline on that host. The latter reads flows from the queue and sends it into elasticsearch. </p><p>This can be easily replicated with the following configuration, for example, though you&#x27;ll need one for each Rabbit queue/sensor/index. A mapping template can be specified here.</p><p>Naturally the hosts for rabbit and elastic will need to be updated accordingly.</p><p>Note: This should be updated to use a mapping template, for setting field types, and possibly using es_doc_id as the elasticsearch document id. As of Nov 2020, we are moving toward saving all fields as type &quot;keyword&quot;, &quot;float&quot;, or &quot;long&quot; only. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">input {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  rabbitmq {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    host =&gt; &#x27;localhost&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    user =&gt; &#x27;guest&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    password =&gt; &quot;${rabbitmq_pass}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    exchange =&gt; &#x27;netsage.direct&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    key =&gt;   XXXXXXX&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    queue =&gt; &#x27;netsage&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    durable =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    subscription_retry_interval_seconds =&gt; 5</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    connection_timeout =&gt; 10000</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">filter {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  if [@metadata][rabbitmq_properties][timestamp] {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    date {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      match =&gt; [&quot;[@metadata][rabbitmq_properties][timestamp]&quot;, &quot;UNIX&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/bb237b1e.9e57bcc8.js"></script>
+</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">output {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      hosts =&gt; [</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME1&quot;,</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          &quot;https://CHANGEME2&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      ]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      user =&gt; &quot;logstash&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      password =&gt; &quot;${logstash_elasticsearch_password}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      cacert =&gt; &quot;/etc/logstash/ca.crt&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      index =&gt; &quot;om-ns-netsage&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      template_overwrite =&gt; true</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      failure_type_logging_whitelist =&gt; []</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      action =&gt; index</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      #ssl_certificate_verification =&gt; false</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    }</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/bb237b1e.ea30ebd1.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/nfdump/index.html b/docs/1.2.7/pipeline/nfdump/index.html
index 6752efa7..0e5d3d24 100644
--- a/docs/1.2.7/pipeline/nfdump/index.html
+++ b/docs/1.2.7/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Nfdump | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Nfdump | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/42f7ba93.8e66f0a3.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/42f7ba93.e851b609.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.7/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/42f7ba93.8e66f0a3.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.7/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/42f7ba93.e851b609.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.7/pipeline/tstat/index.html b/docs/1.2.7/pipeline/tstat/index.html
index c18f3ec7..910d97bc 100644
--- a/docs/1.2.7/pipeline/tstat/index.html
+++ b/docs/1.2.7/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.7"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.7"><meta data-react-helmet="true" property="og:title" content="Tstat | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage Usage"><meta data-react-helmet="true" property="og:description" content="Netsage Usage"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.33482c0f.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e186625e.3f1e3436.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/906738b8.3a283baa.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e186625e.6c1b54b7.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/906738b8.33482c0f.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e186625e.3f1e3436.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.7/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.7</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.7</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.7/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/906738b8.3a283baa.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e186625e.6c1b54b7.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/components/docker_env/index.html b/docs/1.2.8/components/docker_env/index.html
index 02a1a3b7..6419ec47 100644
--- a/docs/1.2.8/components/docker_env/index.html
+++ b/docs/1.2.8/components/docker_env/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Please copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Please copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/abbf3a21.97ae394b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/abbf3a21.2995f8fb.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;RNDNet New York Sflow&quot; and &quot;RNDNet Boston Netflow&quot; or &quot;rtr.one.rndnet.edu&quot; and &quot;rtr.two.nrdnet.edu&quot;. Whatever makes sense in your situation.</p></div></div><ul><li>If you don&#x27;t set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. </li><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><p>Other settings of note in this file include the following. You will not necessarily need to change these, but be aware.</p><p><strong>rabbit_output_host</strong>: this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on <code>rabbit</code>, ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name).</p><p>The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.
 (See comments in the *-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
-on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/abbf3a21.97ae394b.js"></script>
+on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/abbf3a21.2995f8fb.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/components/docker_first_steps/index.html b/docs/1.2.8/components/docker_first_steps/index.html
index 9ae7512c..ebdb5d35 100644
--- a/docs/1.2.8/components/docker_first_steps/index.html
+++ b/docs/1.2.8/components/docker_first_steps/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9bfdba8c.baa4761b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9bfdba8c.b2391bbd.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9bfdba8c.baa4761b.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9bfdba8c.b2391bbd.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/components/docker_pipeline/index.html b/docs/1.2.8/components/docker_pipeline/index.html
index fb6bf24e..78c2179f 100644
--- a/docs/1.2.8/components/docker_pipeline/index.html
+++ b/docs/1.2.8/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/42f0e773.55a7a6c7.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/42f0e773.73889b7e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/42f0e773.55a7a6c7.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/42f0e773.73889b7e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/components/docker_upgrade/index.html b/docs/1.2.8/components/docker_upgrade/index.html
index 06472978..d0d92070 100644
--- a/docs/1.2.8/components/docker_upgrade/index.html
+++ b/docs/1.2.8/components/docker_upgrade/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_upgrade | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="docker_upgrade | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Shut things down"><meta data-react-helmet="true" property="og:description" content="Shut things down"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/a55d7fc8.3675ba14.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/a55d7fc8.d2b40c2a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker_compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker_compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
 <code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-files"></a>Check/Update Files<a class="hash-link" href="#checkupdate-files" title="Direct link to heading">#</a></h3><ul><li><p>Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: &quot;3.7&quot; at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.</p></li><li><p>In the same files, see if the version of nfdump has changed. Look for lines like &quot;image: netsage/nfdump-collector:1.6.18&quot;. If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)
-Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this stop!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-docker-containers"></a>Restart Docker Containers<a class="hash-link" href="#restart-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-files" class="table-of-contents__link">Check/Update Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-docker-containers" class="table-of-contents__link">Restart Docker Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/a55d7fc8.3675ba14.js"></script>
+Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this stop!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-docker-containers"></a>Restart Docker Containers<a class="hash-link" href="#restart-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-files" class="table-of-contents__link">Check/Update Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-docker-containers" class="table-of-contents__link">Restart Docker Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/a55d7fc8.d2b40c2a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/deploy/bare_metal_install/index.html b/docs/1.2.8/deploy/bare_metal_install/index.html
index abe2207b..9917a19f 100644
--- a/docs/1.2.8/deploy/bare_metal_install/index.html
+++ b/docs/1.2.8/deploy/bare_metal_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/49703371.0ed6dd81.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/49703371.0286d91d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;sensor&gt;Netflow Sensor 1&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Flow type - sflow or netflow (defaults to netflow) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -39,16 +39,16 @@
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/49703371.0ed6dd81.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/49703371.0286d91d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/deploy/choose_install/index.html b/docs/1.2.8/deploy/choose_install/index.html
index f4e40de1..eecf6337 100644
--- a/docs/1.2.8/deploy/choose_install/index.html
+++ b/docs/1.2.8/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing Install | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Choosing Install | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="BareMetal or Server Install"><meta data-react-helmet="true" property="og:description" content="BareMetal or Server Install"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d2c4f39e.b83f8cc2.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d2c4f39e.d6c2c200.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d2c4f39e.b83f8cc2.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d2c4f39e.d6c2c200.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/deploy/docker_install_advanced/index.html b/docs/1.2.8/deploy/docker_install_advanced/index.html
index 34e94791..f8ce05d4 100644
--- a/docs/1.2.8/deploy/docker_install_advanced/index.html
+++ b/docs/1.2.8/deploy/docker_install_advanced/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><meta data-react-helmet="true" property="og:description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/56776a50.28321f6f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/56776a50.009ef339.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p><em>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-edit-docker-composeoverrideyml"></a>1. Edit docker-compose.override.yml<a class="hash-link" href="#1-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p><em>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-edit-docker-composeoverrideyml"></a>1. Edit docker-compose.override.yml<a class="hash-link" href="#1-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
 router should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change the last part of the path to something meaningful.</li></ul><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2--edit-netsage_overridexml"></a>2.  Edit netsage_override.xml<a class="hash-link" href="#2--edit-netsage_overridexml" title="Direct link to heading">#</a></h3><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; it will be replaced with a value set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-edit-environment-file"></a>3. Edit environment file<a class="hash-link" href="#3-edit-environment-file" title="Direct link to heading">#</a></h3><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=Example New York sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-new-collector"></a>Running the new collector<a class="hash-link" href="#running-the-new-collector" title="Direct link to heading">#</a></h3><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-the-culling-of-nfcapd-files"></a>To Change the Culling of Nfcapd Files<a class="hash-link" href="#to-change-the-culling-of-nfcapd-files" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk don&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">7</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="for-tstat-data"></a>For Tstat Data<a class="hash-link" href="#for-tstat-data" title="Direct link to heading">#</a></h2><p>Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline&#x27;s ingest rabbit queue (named &quot;netsage_deidentifier_raw&quot;).  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.</p><p>Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See <a href="https://github.com/netsage-project/tstat-transport.git" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/tstat-transport.git</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If you need to modify the amount of memory logstash can use or any other java settings,
 rename the provided example for JVM Options and tweak the settings as desired.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="for-data-saved-to-an-nfs-volume"></a>For Data Saved to an NFS Volume<a class="hash-link" href="#for-data-saved-to-an-nfs-volume" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either</p><ol><li>export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)</li><li>update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml</li></ol><p>Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values.</p><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-edit-docker-composeoverrideyml" class="table-of-contents__link">1. Edit docker-compose.override.yml</a></li><li><a href="#2--edit-netsage_overridexml" class="table-of-contents__link">2.  Edit netsage_override.xml</a></li><li><a href="#3-edit-environment-file" class="table-of-contents__link">3. Edit environment file</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#to-change-the-culling-of-nfcapd-files" class="table-of-contents__link">To Change the Culling of Nfcapd Files</a></li><li><a href="#for-tstat-data" class="table-of-contents__link">For Tstat Data</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#for-data-saved-to-an-nfs-volume" class="table-of-contents__link">For Data Saved to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/56776a50.28321f6f.js"></script>
+You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-edit-docker-composeoverrideyml" class="table-of-contents__link">1. Edit docker-compose.override.yml</a></li><li><a href="#2--edit-netsage_overridexml" class="table-of-contents__link">2.  Edit netsage_override.xml</a></li><li><a href="#3-edit-environment-file" class="table-of-contents__link">3. Edit environment file</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#to-change-the-culling-of-nfcapd-files" class="table-of-contents__link">To Change the Culling of Nfcapd Files</a></li><li><a href="#for-tstat-data" class="table-of-contents__link">For Tstat Data</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li><li><a href="#for-data-saved-to-an-nfs-volume" class="table-of-contents__link">For Data Saved to an NFS Volume</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/56776a50.009ef339.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/deploy/docker_install_simple/index.html b/docs/1.2.8/deploy/docker_install_simple/index.html
index 9220662d..4b7e986a 100644
--- a/docs/1.2.8/deploy/docker_install_simple/index.html
+++ b/docs/1.2.8/deploy/docker_install_simple/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Default Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Docker Default Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/42e094bc.2568d8ec.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/42e094bc.04cc4a66.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-clone-the-netsage-pipeline-project"></a>2. Clone the Netsage Pipeline Project<a class="hash-link" href="#2-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t already, install <a href="https://www.docker.com" target="_blank" rel="noopener noreferrer">Docker</a> and <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">Docker Compose</a> and clone this project</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(If you are upgrading to a new release, see the Upgrade section below!)</p><p>Then checkout the right version of the code.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;tag_name&quot; with the release version you intend to use, e.g., &quot;v1.2.8&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.8.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-create-docker-composeoverrideyml"></a>3. Create Docker-compose.override.yml<a class="hash-link" href="#3-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you only have one collector, you should remove or comment out the section for the collector that is not used.</p></div></div><p>This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to <code>/data/input_data/</code>. Each collector is namespaced by its type so the sflow collector will write data to <code>/data/input_data/sflow/</code> and the netflow collector will write data to <code>/data/input_data/netflow/</code>.  </p><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-environment-file"></a>4. Create Environment File<a class="hash-link" href="#4-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
@@ -29,16 +29,16 @@
 on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p><p></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which version Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier.
 This script will replace the version numbers of docker images in the docker-compose files with the correct values.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h2><p>After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. </p><p>(Or see the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h3><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><p></p><h3>Shut things down</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker_compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3>Update Source Code</h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
 <code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><h3>Check/Update Files</h3><ul><li><p>Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: &quot;3.7&quot; at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.</p></li><li><p>In the same files, see if the version of nfdump has changed. Look for lines like &quot;image: netsage/nfdump-collector:1.6.18&quot;. If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)
-Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3>Update Docker Containers</h3><p>Do not forget this stop!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Restart Docker Containers</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a><ul><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/42e094bc.2568d8ec.js"></script>
+Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3>Update Docker Containers</h3><p>Do not forget this stop!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Restart Docker Containers</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a><ul><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/42e094bc.04cc4a66.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/deploy/docker_troubleshoot/index.html b/docs/1.2.8/deploy/docker_troubleshoot/index.html
index ac1d16e4..71873fa0 100644
--- a/docs/1.2.8/deploy/docker_troubleshoot/index.html
+++ b/docs/1.2.8/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d5140579.770e5402.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d5140579.0f850372.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </p><p>Please see the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d5140579.770e5402.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-flow-issues"></a>Data Flow issues:<a class="hash-link" href="#data-flow-issues" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="resource-limitations"></a>Resource Limitations<a class="hash-link" href="#resource-limitations" title="Direct link to heading">#</a></h3><p>If you are running a lot of data sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </p><p>Please see the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#data-flow-issues" class="table-of-contents__link">Data Flow issues:</a></li><li><a href="#resource-limitations" class="table-of-contents__link">Resource Limitations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d5140579.0f850372.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/devel/dev_dataset/index.html b/docs/1.2.8/devel/dev_dataset/index.html
index facff9d1..1f016f65 100644
--- a/docs/1.2.8/devel/dev_dataset/index.html
+++ b/docs/1.2.8/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e00e17a0.481c92d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e00e17a0.8c02aab3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e00e17a0.481c92d6.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e00e17a0.8c02aab3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/devel/docker_dev_guide/index.html b/docs/1.2.8/devel/docker_dev_guide/index.html
index 40c63de3..ba918e65 100644
--- a/docs/1.2.8/devel/docker_dev_guide/index.html
+++ b/docs/1.2.8/devel/docker_dev_guide/index.html
@@ -6,34 +6,34 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Docker Setup"><meta data-react-helmet="true" property="og:description" content="Docker Setup"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0214960a.e4e7cd3e.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0214960a.7e6a0dc4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
 any git checkout.  Simply follow the directions below and setup your pipeline as instructed.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-base-images"></a>Build Base Images<a class="hash-link" href="#build-base-images" title="Direct link to heading">#</a></h2><p>This is optional.  The image are published on docker hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in this GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuring-the-containers"></a>Configuring the Containers<a class="hash-link" href="#configuring-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t done so already, copy env.example and update it to match your own settings:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbit"></a>Rabbit<a class="hash-link" href="#rabbit" title="Direct link to heading">#</a></h3><p>This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set
 here should be consistent with the config for the logstash and importer </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_ERLANG_COOKIE=&#x27;secret cookie&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBIT_HOST=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_USER=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_PASS=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">discovery.type=single-node</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it&#x27;s set to rabbit</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ <strong>NOTE: Changes will require you to rebuild the container</strong></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h3><p>Define the input rabbit queue.  This should match the importer output queue</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0214960a.e4e7cd3e.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0214960a.7e6a0dc4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/docusaurus/index.html b/docs/1.2.8/docusaurus/index.html
index ff64a01d..3135827a 100644
--- a/docs/1.2.8/docusaurus/index.html
+++ b/docs/1.2.8/docusaurus/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Documentation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Documentation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Using Local Tools"><meta data-react-helmet="true" property="og:description" content="Using Local Tools"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/58d958f4.6a5669a4.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/58d958f4.572280db.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
-at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/58d958f4.6a5669a4.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
+at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/58d958f4.572280db.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/elastic/index.html b/docs/1.2.8/pipeline/elastic/index.html
index 48493810..4bced9da 100644
--- a/docs/1.2.8/pipeline/elastic/index.html
+++ b/docs/1.2.8/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/bed7ab4e.6b5e2485.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/bed7ab4e.fea77ca6.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/bed7ab4e.6b5e2485.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/bed7ab4e.fea77ca6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/importer/index.html b/docs/1.2.8/pipeline/importer/index.html
index 1fe7c869..e863747f 100644
--- a/docs/1.2.8/pipeline/importer/index.html
+++ b/docs/1.2.8/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ad308aff.9d4faea0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ad308aff.8a927dcc.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ad308aff.9d4faea0.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ad308aff.8a927dcc.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/intro/index.html b/docs/1.2.8/pipeline/intro/index.html
index b375c6fa..4037d6ca 100644
--- a/docs/1.2.8/pipeline/intro/index.html
+++ b/docs/1.2.8/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/33f6b1c9.bd4095bc.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/33f6b1c9.798766ac.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/33f6b1c9.bd4095bc.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/33f6b1c9.798766ac.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/logstash/index.html b/docs/1.2.8/pipeline/logstash/index.html
index 702cd4bb..146a1b29 100644
--- a/docs/1.2.8/pipeline/logstash/index.html
+++ b/docs/1.2.8/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6a059f5b.6c7d9748.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6a059f5b.57b36ba4.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 does some data type conversions;
@@ -28,16 +28,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/6a059f5b.6c7d9748.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6a059f5b.57b36ba4.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/nfdump/index.html b/docs/1.2.8/pipeline/nfdump/index.html
index 477b274b..8b68e4ae 100644
--- a/docs/1.2.8/pipeline/nfdump/index.html
+++ b/docs/1.2.8/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Nfdump | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Nfdump | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/b1bfe9f5.13251023.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/b1bfe9f5.592f8cc1.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.8/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/b1bfe9f5.13251023.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.8/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/b1bfe9f5.592f8cc1.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.8/pipeline/tstat/index.html b/docs/1.2.8/pipeline/tstat/index.html
index c7643ec9..2f133216 100644
--- a/docs/1.2.8/pipeline/tstat/index.html
+++ b/docs/1.2.8/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.8"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.8"><meta data-react-helmet="true" property="og:title" content="Tstat | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage Usage"><meta data-react-helmet="true" property="og:description" content="Netsage Usage"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.508016b4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/832ca04e.54bca9dc.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/992819d4.2993e926.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/832ca04e.24044575.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/992819d4.508016b4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/832ca04e.54bca9dc.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.8/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.8</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.8</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.8/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/992819d4.2993e926.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/832ca04e.24044575.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/components/docker_env/index.html b/docs/1.2.9/components/docker_env/index.html
index 62a28fbc..b5ab169f 100644
--- a/docs/1.2.9/components/docker_env/index.html
+++ b/docs/1.2.9/components/docker_env/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Please copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Please copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6125e2e5.da94766a.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6125e2e5.372c6163.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;RNDNet New York Sflow&quot; and &quot;RNDNet Boston Netflow&quot; or &quot;rtr.one.rndnet.edu&quot; and &quot;rtr.two.nrdnet.edu&quot;. Whatever makes sense in your situation.</p></div></div><ul><li>If you don&#x27;t set a sensor name, the default docker hostname, which changes each time you run the pipeline, will be used. </li><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><p>Other settings of note in this file include the following. You will not necessarily need to change these, but be aware.</p><p><strong>rabbit_output_host</strong>: this defines where the final data will land after going through the pipeline.  By default, the last rabbit queue will be on <code>rabbit</code>, ie, the local rabbitMQ server running in its docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name).</p><p>The following Logstash Aggregation Filter settings are exposed in case you wish to use different values.
 (See comments in the *-aggregation.conf file.) The aggregation filter stitches together long-lasting flows that are seen in multiple nfcapd files, matching by the 5-tuple (source and destination IPs, ports, and protocol) plus sensor name. </p><p><strong>Aggregation_maps_path</strong>: the name of the file to which logstash will write in-progress aggregation data when logstash shuts down. When logstash starts up again, it will read this file in and resume aggregating. The filename is configurable for complex situations, but /data/ is required.  </p><p><strong>Inactivity_timeout</strong>: If more than inactivity_timeout seconds have passed between the &#x27;start&#x27; of a flow and the &#x27;start&#x27;
 of the LAST matching flow, OR if no matching flow has coming in for inactivity_timeout seconds
-on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/6125e2e5.da94766a.js"></script>
+on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6125e2e5.372c6163.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/components/docker_first_steps/index.html b/docs/1.2.9/components/docker_first_steps/index.html
index 862b5f32..a20874ee 100644
--- a/docs/1.2.9/components/docker_first_steps/index.html
+++ b/docs/1.2.9/components/docker_first_steps/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/b3c114f4.e41fb427.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/b3c114f4.28ba7bd1.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/b3c114f4.e41fb427.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/b3c114f4.28ba7bd1.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/components/docker_pipeline/index.html b/docs/1.2.9/components/docker_pipeline/index.html
index 41a4f813..6805cead 100644
--- a/docs/1.2.9/components/docker_pipeline/index.html
+++ b/docs/1.2.9/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d1597237.705b0ef9.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d1597237.07f1898e.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d1597237.705b0ef9.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d1597237.07f1898e.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/components/docker_upgrade/index.html b/docs/1.2.9/components/docker_upgrade/index.html
index 2b503724..e7c68120 100644
--- a/docs/1.2.9/components/docker_upgrade/index.html
+++ b/docs/1.2.9/components/docker_upgrade/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_upgrade | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="docker_upgrade | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Shut things down"><meta data-react-helmet="true" property="og:description" content="Shut things down"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/6546ef57.e13c6102.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6546ef57.b480f0cc.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/components/docker_upgrade">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_upgrade">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_upgrade">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">docker_upgrade</h1></header><div class="markdown"><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
 <code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-files"></a>Check/Update Files<a class="hash-link" href="#checkupdate-files" title="Direct link to heading">#</a></h3><ul><li><p>Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: &quot;3.7&quot; at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.</p></li><li><p>In the same files, see if the version of nfdump has changed. Look for lines like &quot;image: netsage/nfdump-collector:1.6.18&quot;. If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)
-Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this step!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-docker-containers"></a>Restart Docker Containers<a class="hash-link" href="#restart-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images"></a>Delete old images<a class="hash-link" href="#delete-old-images" title="Direct link to heading">#</a></h3><p>To save space, delete any old images that are not needed. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-files" class="table-of-contents__link">Check/Update Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-docker-containers" class="table-of-contents__link">Restart Docker Containers</a></li><li><a href="#delete-old-images" class="table-of-contents__link">Delete old images</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/6546ef57.e13c6102.js"></script>
+Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="select-release-version"></a>Select Release Version<a class="hash-link" href="#select-release-version" title="Direct link to heading">#</a></h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>Do not forget this step!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-docker-containers"></a>Restart Docker Containers<a class="hash-link" href="#restart-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images"></a>Delete old images<a class="hash-link" href="#delete-old-images" title="Direct link to heading">#</a></h3><p>To save space, delete any old images that are not needed. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/components/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-files" class="table-of-contents__link">Check/Update Files</a></li><li><a href="#select-release-version" class="table-of-contents__link">Select Release Version</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-docker-containers" class="table-of-contents__link">Restart Docker Containers</a></li><li><a href="#delete-old-images" class="table-of-contents__link">Delete old images</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6546ef57.b480f0cc.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/deploy/bare_metal_install/index.html b/docs/1.2.9/deploy/bare_metal_install/index.html
index 233b2ba0..965f4ee6 100644
--- a/docs/1.2.9/deploy/bare_metal_install/index.html
+++ b/docs/1.2.9/deploy/bare_metal_install/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="NetSage Flow Processing Pipeline Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/56a29d1a.92eaa661.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/56a29d1a.de0b8997.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files. </p><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">NetSage Flow Processing Pipeline Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline on a new machine. Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>Sflow and netflow data and the NetFlow Importer use nfdump tools. If you are only collecting tstat data, you do not need nfdump. </p><p>nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work. Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump.</p><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files. </p><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;sensor&gt;Netflow Sensor 1&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Flow type - sflow or netflow (defaults to netflow) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
@@ -39,16 +39,16 @@
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
 Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/56a29d1a.92eaa661.js"></script>
+Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing Install</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Default Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/56a29d1a.de0b8997.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/deploy/choose_install/index.html b/docs/1.2.9/deploy/choose_install/index.html
index d4b1007d..15b5e975 100644
--- a/docs/1.2.9/deploy/choose_install/index.html
+++ b/docs/1.2.9/deploy/choose_install/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing Install | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Choosing Install | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="BareMetal or Server Install"><meta data-react-helmet="true" property="og:description" content="BareMetal or Server Install"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/f64b3473.09cab9db.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f64b3473.b96c3fd3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/f64b3473.09cab9db.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Choosing Install</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="baremetal-or-server-install"></a>BareMetal or Server Install<a class="hash-link" href="#baremetal-or-server-install" title="Direct link to heading">#</a></h2><p>The baremetal installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, but also has more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The docker version makes it trivial to bring up the pipeline for both a developer and consumer. All the work is mostly already done for you and it&#x27;s should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation</a></li><li><a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Advanced Docker</a> - allows for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">NetSage Flow Processing Pipeline Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#baremetal-or-server-install" class="table-of-contents__link">BareMetal or Server Install</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f64b3473.b96c3fd3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/deploy/docker_install_advanced/index.html b/docs/1.2.9/deploy/docker_install_advanced/index.html
index 9bf9baea..36f15701 100644
--- a/docs/1.2.9/deploy/docker_install_advanced/index.html
+++ b/docs/1.2.9/deploy/docker_install_advanced/index.html
@@ -6,33 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><meta data-react-helmet="true" property="og:description" content="If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/5ee17acf.eeb64f93.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/5ee17acf.4e665490.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p><em>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-edit-docker-composeoverrideyml"></a>1. Edit docker-compose.override.yml<a class="hash-link" href="#1-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Installation Guide</h1></header><div class="markdown"><p>If the Docker Simple installation does not meet your needs, the following customizations will allow for more complex situations.</p><p><em>Please first read the Docker Simple installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-edit-docker-composeoverrideyml"></a>1. Edit docker-compose.override.yml<a class="hash-link" href="#1-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">1.6.18</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector-name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>command: choose between sfcapd for sflow and nfcapd for netflow, and at the end of the command, specify the port to watch for incoming flow data.  (Unless your flow exporter is already set up to use a different port, you can use the default ports and configure the exporters on the routers to match.)</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the
 router should be configured to export data to the same port. (If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change the last part of the path to something meaningful.</li></ul><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2--edit-netsage_overridexml"></a>2.  Edit netsage_override.xml<a class="hash-link" href="#2--edit-netsage_overridexml" title="Direct link to heading">#</a></h3><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; it will be replaced with a value set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-edit-environment-file"></a>3. Edit environment file<a class="hash-link" href="#3-edit-environment-file" title="Direct link to heading">#</a></h3><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=Example New York sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-new-collector"></a>Running the new collector<a class="hash-link" href="#running-the-new-collector" title="Direct link to heading">#</a></h3><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>The default version of the collector is 1.6.18. There are other versions released and :latest should be point to the latest one, but there is no particular effort made to make sure we released the latest version. You can get a listing of all the current tags listed <a href="https://hub.docker.com/r/netsage/nfdump-collector/tags" target="_blank" rel="noopener noreferrer">here</a> and the source to generate the docker image can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">here</a> the code for the You may use a different version though there is no particular effort to have an image for every nfdump release.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to differentiate between flows that enter or exit through specific sensor interfaces. This can be done by editing the env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows through interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name (sensor_id) changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, the number of bits needs to be corrected for the sampling rate. For example, if you are sampling 1 out of 100 flows and a sample has 55 MB, it is assumed that in reality there would be 100 flows of that size (with that src and dst), so the number of bits is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config or the nfcapd command. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk don&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 7 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">7</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-process-tstat-data"></a>To Process Tstat Data<a class="hash-link" href="#to-process-tstat-data" title="Direct link to heading">#</a></h2><p>Tstat data is not collected by nfdump/sfcapd/nfcapd or read by an Importer. Instead, the flow data is sent directly from the router or switch to the logstash pipeline&#x27;s ingest rabbit queue (named &quot;netsage_deidentifier_raw&quot;).  So, when following the Docker Simple guide, the sections related to configuring and starting up the collectors and Importer will not pertain to the tstat sensors. The .env file still needs to be set up though.</p><p>Setting up Tstat is outside the scope of this document, but see the Netsage project Tstat-Transport which contains client programs that can send tstat data to a rabbit queue. See <a href="https://github.com/netsage-project/tstat-transport.git" target="_blank" rel="noopener noreferrer">https://github.com/netsage-project/tstat-transport.git</a>. Basically, you need to have Tstat send data directly to the same rabbit queue that the importers write sflow and netflow data to and that the logstash pipeline reads from.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If you need to modify the amount of memory logstash can use or any other java settings,
 rename the provided example for JVM Options and tweak the settings as desired.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>The recommended heap size for typical ingestion scenarios should be no less than 4GB and no more than 8GB.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="for-data-saved-to-an-nfs-volume"></a>For Data Saved to an NFS Volume<a class="hash-link" href="#for-data-saved-to-an-nfs-volume" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data directory.  If you would like to use an NFS mount instead you will need to either</p><ol><li>export the NFS volume as ${PROJECT_DIR}/data (which is the idea scenario and least intrusive)</li><li>update the path to the NFS export path in all locations in docker-compose.yml and docker-compose.override.yml</li></ol><p>Note: modifying all the paths in the two files should work, but may not. In one case, it worked to modify only the paths for the collector volumes (eg, - /mnt/nfs/netsagedata/netflow:/data), leaving all others with their default values.</p></div></div><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict on upgrade.
 You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.
-:::</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-edit-docker-composeoverrideyml" class="table-of-contents__link">1. Edit docker-compose.override.yml</a></li><li><a href="#2--edit-netsage_overridexml" class="table-of-contents__link">2.  Edit netsage_override.xml</a></li><li><a href="#3-edit-environment-file" class="table-of-contents__link">3. Edit environment file</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/5ee17acf.eeb64f93.js"></script>
+:::</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Default Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a><ul><li><a href="#1-edit-docker-composeoverrideyml" class="table-of-contents__link">1. Edit docker-compose.override.yml</a></li><li><a href="#2--edit-netsage_overridexml" class="table-of-contents__link">2.  Edit netsage_override.xml</a></li><li><a href="#3-edit-environment-file" class="table-of-contents__link">3. Edit environment file</a></li><li><a href="#running-the-new-collector" class="table-of-contents__link">Running the new collector</a></li></ul></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/5ee17acf.4e665490.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/deploy/docker_install_simple/index.html b/docs/1.2.9/deploy/docker_install_simple/index.html
index 6207376c..2741f394 100644
--- a/docs/1.2.9/deploy/docker_install_simple/index.html
+++ b/docs/1.2.9/deploy/docker_install_simple/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Default Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Docker Default Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/c51f4c72.0e8bec24.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/c51f4c72.69c062f8.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Default Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to, by default, netsage-elk1.grnoc.iu.edu)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in this github repo (netsage-project/netsage-pipeline). See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a sensor to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there are collectors (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue on the pipeline host. No collector is needed for tstat data. From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-clone-the-netsage-pipeline-project"></a>2. Clone the Netsage Pipeline Project<a class="hash-link" href="#2-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t already, install <a href="https://www.docker.com" target="_blank" rel="noopener noreferrer">Docker</a> and <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">Docker Compose</a> and clone this project</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(If you are upgrading to a new release, see the Upgrade section below!)</p><p>Then checkout the right version of the code.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout tag_name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;tag_name&quot; with the release version you intend to use, e.g., &quot;v1.2.8&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.8.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-create-docker-composeoverrideyml"></a>3. Create Docker-compose.override.yml<a class="hash-link" href="#3-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single netflow collector and a single sflow collector. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you only have one collector, you should remove or comment out the section for the collector that is not used.</p></div></div><p>This file also specifies port numbers, and directories for nfcapd files.  By default, the sflow collector will listen to udp traffic on localhost:9998, while the netflow collector will listen on port 9999,  and data will be written to <code>/data/input_data/</code>. Each collector is namespaced by its type so the sflow collector will write data to <code>/data/input_data/sflow/</code> and the netflow collector will write data to <code>/data/input_data/netflow/</code>.  </p><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>You may need to remove all the comments in the override file as they may conflict with the parsing done by docker-compose</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-environment-file"></a>4. Create Environment File<a class="hash-link" href="#4-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Please copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=my sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=my netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Simply change the names to unique identifiers (with spaces or not, no quotes) and you&#x27;re good to go. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field and will be shown in Grafana dashboards. Choose names that are meaningful and unique.
@@ -29,16 +29,16 @@
 on the clock, assume the flow has ended.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Nfcapd files are typically written every 5 minutes. Netsage uses an inactivity_timeout = 630 sec = 10.5 min for 5-min files; 960 sec = 16 min for 15-min files.  (For 5-min files, this allows one 5 min gap or period during which the no. of bits transferred don&#x27;t meet the cutoff)</p></div></div><p><strong>max_flow_timeout</strong>: If a long-lasting flow is still aggregating when this timeout is reached, arbitrarily cut it off and start a new flow.  The default is 24 hours.</p><p></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which version Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier.
 This script will replace the version numbers of docker images in the docker-compose files with the correct values.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors"></a>Running the Collectors<a class="hash-link" href="#running-the-collectors" title="Direct link to heading">#</a></h2><p>After selecting the version to run, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command. </p><p>(Or see the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If the collector(s) are running properly, you should see nfcapd files in subdirectories of data/input_data/, and they should have sizes of more than a few hundred bytes. (See Troubleshooting if you have problems.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h3><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="upgrading"></a>Upgrading<a class="hash-link" href="#upgrading" title="Direct link to heading">#</a></h2><p></p><h3>Shut things down</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3>Update Source Code</h3><p>To upgrade to a new release, just reset and pull changes including the new release from github. Your customized .env and override files will not be overwritten.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p><p>Example:
 <code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><h3>Check/Update Files</h3><ul><li><p>Compare the new docker-compose.override_example.yml file to your docker-compose.override.yml to see if a new version of Docker is required. Look for, eg, version: &quot;3.7&quot; at the top. If the version number is different, change it in your docker-compose.override.yml file and upgrade Docker manually.</p></li><li><p>In the same files, see if the version of nfdump has changed. Look for lines like &quot;image: netsage/nfdump-collector:1.6.18&quot;. If there has been a change, update the version in the override file. (You do not need to actually perform any update yourself.)
-Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3>Update Docker Containers</h3><p>Do not forget this step!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Restart Docker Containers</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3>Delete old images</h3><p>To save space, delete any old images that are not needed. </p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a><ul><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/c51f4c72.0e8bec24.js"></script>
+Note that you do not need to update the versions of the importer or logstash images. That will be done for you in the &quot;select release version&quot; stop coming up.</p></li><li><p>Also compare your .env file with the new env.example file to see if any new lines or sections have been added. Copy new lines into your .env file, making any appropriate changes to example values.</p></li><li><p>If you used the Docker Advanced guide to make a netsage_override.xml file, compare it to netsage_shared.xml to see if there are any changes. This is unlikely.</p></li></ul><h3>Select Release Version</h3><p>Run these two commands to select the new release you want to run. In the first, replace &quot;tag_value&quot; by the version to run (eg, v1.2.8). When asked by the second, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b tag_value </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Check to be sure docker-compose.yml and docker-compose.override.yml both now have the version number you selected.  </p><h3>Update Docker Containers</h3><p>Do not forget this step!  This applies for both development and release versions.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3>Restart Docker Containers</h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3>Delete old images</h3><p>To save space, delete any old images that are not needed. </p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« NetSage Flow Processing Pipeline Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-clone-the-netsage-pipeline-project" class="table-of-contents__link">2. Clone the Netsage Pipeline Project</a></li><li><a href="#3-create-docker-composeoverrideyml" class="table-of-contents__link">3. Create Docker-compose.override.yml</a></li><li><a href="#4-create-environment-file" class="table-of-contents__link">4. Create Environment File</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#running-the-collectors" class="table-of-contents__link">Running the Collectors</a><ul><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></li><li><a href="#upgrading" class="table-of-contents__link">Upgrading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/c51f4c72.69c062f8.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/deploy/docker_troubleshoot/index.html b/docs/1.2.9/deploy/docker_troubleshoot/index.html
index 281d5693..fbcd441c 100644
--- a/docs/1.2.9/deploy/docker_troubleshoot/index.html
+++ b/docs/1.2.9/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1d038b8a.35918ded.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1d038b8a.936f367a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Logstash or Importer errors:</strong></p><ul><li>Make sure all containers are running.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the containers are running but nothing is happening and the disk is full, you probably need more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </li></ul><p>Please see the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/1d038b8a.35918ded.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, day and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file.</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully. </li><li>If the final rabbit queue is on an external host, check iptables on that host to be sure incoming traffic from your pipeline host is allowed.</li></ul><p>To see if flows are getting into and being read from the rabbit queue on the pipeline host, you can go to  <code>http://localhost:15672</code> in your favorite web browser. Login as guest with password guest. Look for accumulating messages and/or messages being acknowledged and published.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Logstash or Importer errors:</strong></p><ul><li>Make sure all containers are running.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs -f $service_label</code></li><li>Check the logs to see if logstash is starting successfully.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the containers are running but nothing is happening and the disk is full, you probably need more disk space. You could try saving fewer days of nfcapd files (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash which is usually only allocated 2GB of RAM. You&#x27;ll need to update the jvm.options file to grant it more memory. </li></ul><p>Please see the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced#customize-logstash-settings">Docker Advanced guide</a> for details on how to customize logstash.</p><p>Applying this snippet to logstash may help. For example, to give logstash (java) 3GB,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">environment</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> + LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx3g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Alternatively you may also try doing this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">deploy</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">resources</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">limits</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.50&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 50M</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">reservations</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">cpus</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;0.25&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token key atrule">memory</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> 20M</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Reference: <a href="https://docs.docker.com/compose/compose-file/#resources" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/compose-file/#resources</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/1d038b8a.936f367a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/devel/dev_dataset/index.html b/docs/1.2.9/devel/dev_dataset/index.html
index 6b0cb79d..da9de3bc 100644
--- a/docs/1.2.9/devel/dev_dataset/index.html
+++ b/docs/1.2.9/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/40b960a4.2c1c192b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/40b960a4.422649da.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/40b960a4.2c1c192b.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/40b960a4.422649da.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/devel/docker_dev_guide/index.html b/docs/1.2.9/devel/docker_dev_guide/index.html
index 87ed382f..eb5bf248 100644
--- a/docs/1.2.9/devel/docker_dev_guide/index.html
+++ b/docs/1.2.9/devel/docker_dev_guide/index.html
@@ -6,34 +6,34 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Docker Setup"><meta data-react-helmet="true" property="og:description" content="Docker Setup"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9b505a1f.0d4ca405.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9b505a1f.f4b85bdc.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-setup"></a>Docker Setup<a class="hash-link" href="#docker-setup" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configure-collectors"></a>Configure Collectors<a class="hash-link" href="#configure-collectors" title="Direct link to heading">#</a></h2><p>Before you start if you haven&#x27;t already done so, please make a copy of docker-compose.override_example.yml this is used to setup your collectors.  The default should work out of the box with the env.example provided.  If you wish you add customizations, please see the <a href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced Install Guide</a></p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>We currently release a development version and a tagged version.  The first version to support will be. v1.2.5 once release.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stable-release-version"></a>Stable Release Version<a class="hash-link" href="#stable-release-version" title="Direct link to heading">#</a></h3><p>To select a released version please do the following.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> scripts/docker_select_version.sh tagValue </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Example:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">scripts/docker_select_version.sh v1.2.5 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run the script without any version specified it&#x27;ll list all the current tags and prompt you to select a version.</p><p>Once that&#x27;s complete, all the instructions below are still applicable. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="development-version"></a>Development Version<a class="hash-link" href="#development-version" title="Direct link to heading">#</a></h3><p>If you wish to use the development version you are free to do so.  It is the default behavior on
 any git checkout.  Simply follow the directions below and setup your pipeline as instructed.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-base-images"></a>Build Base Images<a class="hash-link" href="#build-base-images" title="Direct link to heading">#</a></h2><p>This is optional.  The image are published on docker hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in this GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuring-the-containers"></a>Configuring the Containers<a class="hash-link" href="#configuring-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="environment-file"></a>Environment File<a class="hash-link" href="#environment-file" title="Direct link to heading">#</a></h3><p>If you haven&#x27;t done so already, copy env.example and update it to match your own settings:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="rabbit"></a>Rabbit<a class="hash-link" href="#rabbit" title="Direct link to heading">#</a></h3><p>This portion is primarily to set the Rabbit MQ server.  Most of the default settings work but whatever values you set
 here should be consistent with the config for the logstash and importer </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_ERLANG_COOKIE=&#x27;secret cookie&#x27;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBIT_HOST=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_USER=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">RABBITMQ_DEFAULT_PASS=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">discovery.type=single-node</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Note the hostname will follow the docker-compose label.  You can rename it if you like but by default it&#x27;s set to rabbit</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h3><p>The importer config is defined in compose/netsage_shared.xml.  If you use different values then the defaults you may want to change them/ <strong>NOTE: Changes will require you to rebuild the container</strong></p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash"></a>Logstash<a class="hash-link" href="#logstash" title="Direct link to heading">#</a></h3><p>Define the input rabbit queue.  This should match the importer output queue</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_input_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9b505a1f.0d4ca405.js"></script>
+</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Define the output rabbit queue.  This can be the docker container or any valid RabbitMQ server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h3><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-containers"></a>Running the Containers<a class="hash-link" href="#running-the-containers" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Documentation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configure-collectors" class="table-of-contents__link">Configure Collectors</a></li><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a><ul><li><a href="#stable-release-version" class="table-of-contents__link">Stable Release Version</a></li><li><a href="#development-version" class="table-of-contents__link">Development Version</a></li></ul></li><li><a href="#build-base-images" class="table-of-contents__link">Build Base Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#configuring-the-containers" class="table-of-contents__link">Configuring the Containers</a><ul><li><a href="#environment-file" class="table-of-contents__link">Environment File</a></li><li><a href="#rabbit" class="table-of-contents__link">Rabbit</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#logstash" class="table-of-contents__link">Logstash</a></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li></ul></li><li><a href="#running-the-containers" class="table-of-contents__link">Running the Containers</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9b505a1f.f4b85bdc.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/docusaurus/index.html b/docs/1.2.9/docusaurus/index.html
index 04c3995b..506a1d3f 100644
--- a/docs/1.2.9/docusaurus/index.html
+++ b/docs/1.2.9/docusaurus/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Documentation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Documentation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Using Local Tools"><meta data-react-helmet="true" property="og:description" content="Using Local Tools"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/db0ed613.02dc005f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/db0ed613.c9712562.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
-at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/db0ed613.02dc005f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/docusaurus">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/docusaurus">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/docusaurus">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/docusaurus">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/docusaurus">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Documentation Guide</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-local-tools"></a>Using Local Tools<a class="hash-link" href="#using-local-tools" title="Direct link to heading">#</a></h1><p>This project&#x27;s documentation is using docusaurus to generate docs from markdown files and loaded in a react app.</p><p>To extend the doc simply create a markdown file and reference the ID in the side bar. Please see the related documentation
+at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development"></a>Local Development<a class="hash-link" href="#local-development" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build"></a>Build<a class="hash-link" href="#build" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This command generates static content into the <code>build</code> directory and can be served using any static contents hosting service.</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="using-docker"></a>Using Docker<a class="hash-link" href="#using-docker" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-the-container"></a>Build the container<a class="hash-link" href="#build-the-container" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose build build_docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="local-development-1"></a>Local Development<a class="hash-link" href="#local-development-1" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compopse up -d docs</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docs will be visible at http://localhost:8000/netsage-pipeline/</p><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-new-release"></a>Tagging New release<a class="hash-link" href="#tagging-new-release" title="Direct link to heading">#</a></h1><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">yarn run docusaurus docs:version 1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Where 1.1.0 is the next release version.  Commit the new directories created under </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Alternatively, if you have docker setup, you may run this command and avoid installing all of the required javascript dependencies.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose run  docs yarn run docusaurus docs:version  1.1.0</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></div><p>(note if image is not built yet, please run: <code>docker-compose build build_docs</code>)</p><ul><li>website/versioned_docs</li><li>website/versioned_sidebars</li><li>versions.json </li></ul><p>Update the docusaurus.config.js</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">lastVersion: &#x27;1.2.6&#x27; </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>should point to the latest value.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="removing-a-version"></a>Removing a version<a class="hash-link" href="#removing-a-version" title="Direct link to heading">#</a></h3><p>To remove version 1.2.6 for example.</p><p>we need to: </p><ul><li>update versions.json to remove the reference</li><li>remove the versioned_docs/version-1.2.6</li><li>remove versioned_sidebars/version-1.2.6-sidebars.json</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#local-development" class="table-of-contents__link">Local Development</a></li><li><a href="#build" class="table-of-contents__link">Build</a></li><li><a href="#build-the-container" class="table-of-contents__link">Build the container</a></li><li><a href="#local-development-1" class="table-of-contents__link">Local Development</a><ul><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/db0ed613.c9712562.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/elastic/index.html b/docs/1.2.9/pipeline/elastic/index.html
index 9561f202..32dccc0a 100644
--- a/docs/1.2.9/pipeline/elastic/index.html
+++ b/docs/1.2.9/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/5a20f281.847d72f0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/5a20f281.84ff2815.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/5a20f281.847d72f0.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f05420d7ded9eb151ccd293a633 ff226d1752b24e0f4139a87a8b26d779</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>Essentially time the flow went into the logstash pipeline or the time stitching of the flow commenced</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow went into the logstash pipeline for tstat flows, or the time stitching finished and the event was pushed for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id. If es_doc_id is provided, that is used.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing Install »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/5a20f281.84ff2815.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/importer/index.html b/docs/1.2.9/pipeline/importer/index.html
index ad6bb2c1..044d124a 100644
--- a/docs/1.2.9/pipeline/importer/index.html
+++ b/docs/1.2.9/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/99d6017e.6764251a.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/99d6017e.cb54715a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/99d6017e.6764251a.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><p>NOTE: The importer will be deprecated in the future and replace with a logstash operation.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Nfdump</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/99d6017e.cb54715a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/intro/index.html b/docs/1.2.9/pipeline/intro/index.html
index a0b64c3b..16effe0d 100644
--- a/docs/1.2.9/pipeline/intro/index.html
+++ b/docs/1.2.9/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/c3f49368.7fcf56c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/c3f49368.e29a1244.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/c3f49368.7fcf56c0.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components (currently)</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Server&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the &quot;Docker Simple&quot; Install should suffice. The &quot;Docker Advanced&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/c3f49368.e29a1244.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/logstash/index.html b/docs/1.2.9/pipeline/logstash/index.html
index 1c71178b..f66c28d3 100644
--- a/docs/1.2.9/pipeline/logstash/index.html
+++ b/docs/1.2.9/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.)"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9720d299.d4ac0286.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9720d299.9abe8149.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows (normally from a RabbitMQ queue), performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config.  (In principle, Logstash can run more than one &quot;logstash pipeline&quot; at a time, but that will not be discussed here.) </p><p>Logstash config files invoke various logstash &quot;filters&quot;. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a .conf file are not needed in your particular case, it can be removed or disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry Database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 does some data type conversions;
@@ -28,16 +28,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9720d299.d4ac0286.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9720d299.9abe8149.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/nfdump/index.html b/docs/1.2.9/pipeline/nfdump/index.html
index 0c211c39..ebd88b12 100644
--- a/docs/1.2.9/pipeline/nfdump/index.html
+++ b/docs/1.2.9/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Nfdump | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Nfdump | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7b85b03c.95489b8b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7b85b03c.6e3a64bd.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.9/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/7b85b03c.95489b8b.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Nfdump</h1></header><div class="markdown"><p>Nfdump is a toolset used to collect and process netflow and sflow data that is sent from netflow/sflow compatible devices. The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW. Nfdump supports IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>The nfdump utility (nfcapd and/or sfcapd processes) is used to collect incoming netflow and sflow data and save it to disk (as nfcapd files).  The files are then processed by the <a href="/netsage-pipeline/docs/1.2.9/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump processes can be invoked locally or using a Docker container.  The Docker Deployment Guide walks you through utilizing the Docker container.  The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/7b85b03c.6e3a64bd.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.2.9/pipeline/tstat/index.html b/docs/1.2.9/pipeline/tstat/index.html
index f41ce599..4eca8d0c 100644
--- a/docs/1.2.9/pipeline/tstat/index.html
+++ b/docs/1.2.9/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="1.2.9"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-1.2.9"><meta data-react-helmet="true" property="og:title" content="Tstat | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage Usage"><meta data-react-helmet="true" property="og:description" content="Netsage Usage"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.84f0767e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/50032990.0ea61ccd.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d75898ab.8089cc99.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/50032990.47d0b7f0.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/d75898ab.84f0767e.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/50032990.0ea61ccd.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">Tstat</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">Nfdump</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">Server Installation Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">Docker Simple</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">Docker Advanced</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment Guides</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">Docker Dev Guide</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Documentation</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/1.2.9/docusaurus">Documentation Guide</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for Netsage Pipeline Documentation <strong>1.2.9</strong>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 1.2.9</span></div><header><h1 class="docTitle_3a4h">Tstat</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage project <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> provides a way to send the captured data to a rabbitmq host where it can then be processed by a <a href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Docker images exist for the tstat process, as well as the sending of tstat logs to the appropriate RabbitMQ server.  This flow is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-1.2.9/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Nfdump »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/d75898ab.8089cc99.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/50032990.47d0b7f0.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/components/docker_env/index.html b/docs/components/docker_env/index.html
index e554114e..76f445c4 100644
--- a/docs/components/docker_env/index.html
+++ b/docs/components/docker_env/index.html
@@ -5,31 +5,31 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Next, copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Next, copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9b6d3056.ff7822d9.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Next, copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Next, copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/8b301107.d02fcb00.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
-For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9b6d3056.ff7822d9.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
+For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/8b301107.d02fcb00.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/components/docker_first_steps/index.html b/docs/components/docker_first_steps/index.html
index ad124ca4..4cc6056e 100644
--- a/docs/components/docker_first_steps/index.html
+++ b/docs/components/docker_first_steps/index.html
@@ -5,33 +5,33 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/a3c17fac.4f9a964b.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ff1d9e71.c2bc6db2.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/a3c17fac.4f9a964b.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ff1d9e71.c2bc6db2.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/components/docker_pipeline/index.html b/docs/components/docker_pipeline/index.html
index 935eae5d..912e7c74 100644
--- a/docs/components/docker_pipeline/index.html
+++ b/docs/components/docker_pipeline/index.html
@@ -5,30 +5,30 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/89d5514b.c223f8ef.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6e2a1c72.7c78a48b.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/89d5514b.c223f8ef.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6e2a1c72.7c78a48b.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/deploy/bare_metal_install/index.html b/docs/deploy/bare_metal_install/index.html
deleted file mode 100644
index d2f64aa9..00000000
--- a/docs/deploy/bare_metal_install/index.html
+++ /dev/null
@@ -1,56 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width,initial-scale=1">
-<meta name="generator" content="Docusaurus v2.0.0-alpha.72">
-<link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Manual Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Manual Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/b56f03c6.3836580d.js" as="script">
-</head>
-<body>
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
-The Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. </p><p>Nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work.
-Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included.</p></div></div><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;sensor&gt;Netflow Sensor 1&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Flow type - sflow or netflow (defaults to netflow) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &quot;instance&quot; goes along with sensor.  This is to identify various instances if a sensor has --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- more than one &quot;stream&quot; / data collection.  Defaults to 0. --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &lt;instance&gt;1&lt;/instance&gt; --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &lt;router-address&gt;&lt;/router-address&gt; --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;/collection&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. </p><p>There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) </p><p>Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL.</p><p>If you&#x27;re running a default RabbitMQ config, which is open only to &#x27;localhost&#x27; as guest/guest, you won&#x27;t need to change anything here.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;!-- rabbitmq connection info --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;rabbit_input&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;host&gt;127.0.0.1&lt;/host&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;port&gt;5672&lt;/port&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;username&gt;guest&lt;/username&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;password&gt;guest&lt;/password&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;ssl&gt;0&lt;/ssl&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;batch_size&gt;100&lt;/batch_size&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;vhost&gt;/&lt;/vhost&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;durable&gt;1&lt;/durable&gt; &lt;!-- Whether the rabbit queue is &#x27;durable&#x27; (don&#x27;t change this unless you have a reason) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/rabbit_input&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;rabbit_output&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;host&gt;127.0.0.1&lt;/host&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;port&gt;5672&lt;/port&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;username&gt;guest&lt;/username&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;password&gt;guest&lt;/password&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;ssl&gt;0&lt;/ssl&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;batch_size&gt;100&lt;/batch_size&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;vhost&gt;/&lt;/vhost&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;durable&gt;1&lt;/durable&gt; &lt;!-- Whether the rabbit queue is &#x27;durable&#x27; (don&#x27;t change this unless you have a reason) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/rabbit_output&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-importer-config-file"></a>Setting up the Importer config file<a class="hash-link" href="#setting-up-the-importer-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml</code></p><p>This file has a few more setting specific to the Importer component which you may like to adjust.  </p><ul><li>Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  </li><li>(The Importer does not actually use an input rabbit queue, so we add a &quot;fake&quot; one here.)</li><li>Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded.</li><li>Min-file-age is used to be sure files are complete before being read. </li><li>Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. </li><li>Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file.</li><li>Keep num-processes set to 1.</li></ul><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--  NOTE: Values here override those in the shared config --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- rabbitmq queues --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_input</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">netsage_deidentifier_netflow_fake</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">2</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_input</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_output</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">3</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">netsage_deidentifier_raw</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_output</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- How many flows to process at once --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-batch-size</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">100</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-batch-size</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- How many concurrent workers should perform the necessary operations --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">num-processes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">num-processes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- path to nfdump executable (defaults to /usr/bin/nfdump) --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--   &lt;nfdump-path&gt;/path/to/nfdump&lt;/nfdump-path&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Where to store the cache, where it tracks what files it has/hasn&#x27;t read --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cache-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/cache/netsage/netflow_importer.cache</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cache-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- The minium flow size threshold - will not  import any flows smaller than this --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Defaults to 500M  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">min-bytes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">100000000</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">min-bytes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"> </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Do not import nfcapd files younger than min-file-age</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        The value must match /^(\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        &quot;day(s)&quot;, &quot;week(s)&quot;, &quot;month(s)&quot;, &quot;year(s)&quot;, &quot;hour(s)&quot;, &quot;minute(s)&quot; and &quot;second(s)&quot;, respectively&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        Default: 0 (no minimum age) </span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">    --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">min-file-age</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">10m</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">min-file-age</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"> </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- cull-enable: whether to cull processed flow data files --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- default: no culling; set to 1 to turn culling on --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--    &lt;cull-enable&gt;1&lt;/cull-enable&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- cull-tty: cull time to live, in days --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- number of days to retain imported data files before deleting them; default: 3 --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--    &lt;cull-ttl&gt;5&lt;/cull-ttl&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
-Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/b56f03c6.3836580d.js"></script>
-</body>
-</html>
\ No newline at end of file
diff --git a/docs/deploy/choose_install/index.html b/docs/deploy/choose_install/index.html
deleted file mode 100644
index c474df85..00000000
--- a/docs/deploy/choose_install/index.html
+++ /dev/null
@@ -1,34 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width,initial-scale=1">
-<meta name="generator" content="Docusaurus v2.0.0-alpha.72">
-<link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing an Installation Procedure | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Choosing an Installation Procedure | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Manual or BareMetal Installation"><meta data-react-helmet="true" property="og:description" content="Manual or BareMetal Installation"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d2c255b4.51d1c434.js" as="script">
-</head>
-<body>
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d2c255b4.51d1c434.js"></script>
-</body>
-</html>
\ No newline at end of file
diff --git a/docs/deploy/docker_install_advanced/index.html b/docs/deploy/docker_install_advanced/index.html
index 883776fb..e6b0db85 100644
--- a/docs/deploy/docker_install_advanced/index.html
+++ b/docs/deploy/docker_install_advanced/index.html
@@ -5,35 +5,34 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Options Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Options Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><meta data-react-helmet="true" property="og:description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/00cb704f.d8fb9eb4.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Options Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Options Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><meta data-react-helmet="true" property="og:description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9f628d52.08c37d03.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
 In the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.
 In the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.
-In the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  </p><p>Spaces don&#x27;t matter except within the sensor names. Punctuation is required as shown.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-filter-flows-by-subnet"></a>To Filter Flows by Subnet<a class="hash-link" href="#to-filter-flows-by-subnet" title="Direct link to heading">#</a></h2><p>With this option, flows from specified sensors will be dropped unless src or dst is in the list of subnets to keep. It works similarly to the option to filter by interface.  &quot;ALL&quot; can refer to all sensors.
-If a sensor is not referenced at all, all of its flows will be kept. </p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">subnet_filter_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">subnet_filter_keep=Sensor A Name: 123.45.6.0/16; Sensor B Name: 123.33.33.0/24, 456.66.66.0/24</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk doesn&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-save-flow-data-to-a-different-location"></a>To Save Flow Data to a Different Location<a class="hash-link" href="#to-save-flow-data-to-a-different-location" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options.</p><ol><li>The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.</li></ol><p>During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline dir}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir /var/netsage</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rm data/.placeholder</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rmdir data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ln -s /var/netsage {netsage-pipeline dir}/data</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(Check the permissions of the directory.)</p><ol start="2"><li>Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to <code>- /mydir/input_data/netflow:/data</code> (similarly for sflow) and set the importer and logstash volumes to <code>- /mydir:/data</code>. </li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-filter-flows-by-subnet" class="table-of-contents__link">To Filter Flows by Subnet</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/00cb704f.d8fb9eb4.js"></script>
+In the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  </p><p>Spaces don&#x27;t matter except within the sensor names. Punctuation is required as shown.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk doesn&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-save-flow-data-to-a-different-location"></a>To Save Flow Data to a Different Location<a class="hash-link" href="#to-save-flow-data-to-a-different-location" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options.</p><ol><li>The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.</li></ol><p>During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline dir}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir /var/netsage</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rm data/.placeholder</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rmdir data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ln -s /var/netsage {netsage-pipeline dir}/data</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(Check the permissions of the directory.)</p><ol start="2"><li>Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to <code>- /mydir/input_data/netflow:/data</code> (similarly for sflow) and set the importer and logstash volumes to <code>- /mydir:/data</code>. </li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.
+You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9f628d52.08c37d03.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/deploy/docker_install_simple/index.html b/docs/deploy/docker_install_simple/index.html
index dafee28a..1379eba3 100644
--- a/docs/deploy/docker_install_simple/index.html
+++ b/docs/deploy/docker_install_simple/index.html
@@ -5,37 +5,37 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/418618b7.efff362b.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/12ba72b5.4b8388c3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there will be <em>collectors</em> (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input rabbit queue &quot;netsage_deidentifier_raw&quot; on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-set-up-a-pipeline-host"></a>2. Set up a Pipeline Host<a class="hash-link" href="#2-set-up-a-pipeline-host" title="Direct link to heading">#</a></h3><p>Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. </p><p>Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at <a href="https://docs.docker.com/engine/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/engine/install/</a>.</p><p>Install Docker Compose from Docker&#x27;s GitHub repository - see <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/install/</a>.  You need to <strong>specify version 1.29.2</strong> (or newer) in the curl command. </p><p>Check default file permissions. If the <em>logstash</em> user is not able to access the logstash config files in the git checkout, you&#x27;ll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-clone-the-netsage-pipeline-project"></a>3. Clone the Netsage Pipeline Project<a class="hash-link" href="#3-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>Clone the netsage-pipeline project from github.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version.</p><p>Move into the netsage-pipeline/ directory (<strong>all git and docker commands must be run from inside this directory!</strong>), then checkout the most recent version of the code. It will say you are in &#x27;detached HEAD&#x27; state if you don&#x27;t include -b.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout {tag}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;{tag}&quot; with the release version you intend to use, e.g., &quot;v1.2.11&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.11.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-docker-composeoverrideyml"></a>4. Create Docker-compose.override.yml<a class="hash-link" href="#4-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. </p><ul><li>If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</li><li>If the collectors need to listen to different ports, make the appropriate changes here in both the &quot;command:&quot; and &quot;ports:&quot; lines. </li><li>By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in <code>./data/input_data/</code> (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.</li></ul><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem.</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which image versions Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier. </p><p>This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="6-create-environment-file"></a>6. Create Environment File<a class="hash-link" href="#6-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="testing-the-collectors"></a>Testing the Collectors<a class="hash-link" href="#testing-the-collectors" title="Direct link to heading">#</a></h2><p>At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  </p><p>(See the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Subdirectories for sflow/netflow, year, month, and day are created automatically under <code>data/input_data/</code>. File names contain dates and times.
 These are not text files; to view the contents, use an <a href="http://www.linuxcertif.com/man/1/nfdump/" target="_blank" rel="noopener noreferrer">nfdump command</a> (you will need to install nfdump).
 Files will be deleted automatically by the importer as they age out (the default is to keep 3 days).  </p><p>If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)<br>
-See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/418618b7.efff362b.js"></script>
+See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/12ba72b5.4b8388c3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/deploy/docker_troubleshoot/index.html b/docs/deploy/docker_troubleshoot/index.html
index ead3109e..2ab42ebd 100644
--- a/docs/deploy/docker_troubleshoot/index.html
+++ b/docs/deploy/docker_troubleshoot/index.html
@@ -5,31 +5,31 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1bbe794e.484f5992.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/86d09e6e.43e743b3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/1bbe794e.484f5992.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/86d09e6e.43e743b3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/deploy/docker_upgrade/index.html b/docs/deploy/docker_upgrade/index.html
index 9ad29e6e..e1fdb003 100644
--- a/docs/deploy/docker_upgrade/index.html
+++ b/docs/deploy/docker_upgrade/index.html
@@ -5,30 +5,30 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Upgrading | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Upgrading | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><meta data-react-helmet="true" property="og:description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e3649aa7.41406e34.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Upgrading | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Upgrading | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><meta data-react-helmet="true" property="og:description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/9191c864.9b61c372.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these three commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e3649aa7.41406e34.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git fetch</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these commands to upgrade to the current announced version. While older versions of the pipeline are available, they are not recommended or supported for daily use. If in doubt, the Master branch contains the latest version. The documentation version at the top of this page is also the current pipeline version number.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git switch {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./Upgrade_x_x_x.sh (where x_x_x is the current version you are upgrading to)</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/9191c864.9b61c372.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/devel/dev_dataset/index.html b/docs/devel/dev_dataset/index.html
index 88442d2f..0b730e9c 100644
--- a/docs/devel/dev_dataset/index.html
+++ b/docs/devel/dev_dataset/index.html
@@ -5,32 +5,32 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ded423ee.df3ed6d9.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/12ecfc4a.a1b9aa60.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ded423ee.df3ed6d9.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/12ecfc4a.a1b9aa60.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/devel/docker_dev_guide/index.html b/docs/devel/docker_dev_guide/index.html
index 0663e8d1..0f686067 100644
--- a/docs/devel/docker_dev_guide/index.html
+++ b/docs/devel/docker_dev_guide/index.html
@@ -5,33 +5,32 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Selecting a Version"><meta data-react-helmet="true" property="og:description" content="Selecting a Version"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/53b0b5d8.144adf6f.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Selecting a Version"><meta data-react-helmet="true" property="og:description" content="Selecting a Version"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/0dd3dba6.3dbbc1f3.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
 To select a released version use the docker_select_version.sh script (see the Deployment Guide).
-If you wish to use the development version (master branch) simply skip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-images"></a>Build Images<a class="hash-link" href="#build-images" title="Direct link to heading">#</a></h2><p>The images are published on Docker Hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in the GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/53b0b5d8.144adf6f.js"></script>
+If you wish to use the development version (master branch) simply skip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/0dd3dba6.3dbbc1f3.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/devel/docker_dev_tag/index.html b/docs/devel/docker_dev_tag/index.html
index 430c55a9..517fff7e 100644
--- a/docs/devel/docker_dev_tag/index.html
+++ b/docs/devel/docker_dev_tag/index.html
@@ -5,33 +5,33 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">How to Release a New Version of the Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="How to Release a New Version of the Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><meta data-react-helmet="true" property="og:description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e4c61278.03e947b6.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">How to Release a New Version of the Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="How to Release a New Version of the Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><meta data-react-helmet="true" property="og:description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/6e3166bc.fa40017d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
 Be sure to copy info from the CHANGES file into the Release description.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-build-and-push-images-manually"></a>To Build and Push Images Manually<a class="hash-link" href="#to-build-and-push-images-manually" title="Direct link to heading">#</a></h2><p>Below is the procedure to build pipeline_importer and pipeline_logstash images manually.</p><p>Install docker-compose if not done already. See the Docker Installation instructions.</p><p>Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd netsage-pipeline</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b v1.2.11</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh 1.2.11</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     or  $ docker push $image:$tag                        (will push a specific image version)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.
 If that doesn&#x27;t work, try adding this to /etc/hosts: <code>67.219.148.138  mirrorlist.centos.org</code>, and/or try <code>yum install net-tools bridge-utils</code>, and/or restart network.service then docker. </p><p>The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level).</p><p>It might be a good idea to test the images before pushing them. See &quot;Test Docker Images&quot; below.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="building-with-automation"></a>Building With Automation<a class="hash-link" href="#building-with-automation" title="Direct link to heading">#</a></h2><p>???</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="test-docker-images"></a>Test Docker Images<a class="hash-link" href="#test-docker-images" title="Direct link to heading">#</a></h2><p>See the Docker installation instructions for details... </p><p>In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. </p><p>Run docker_select_version.sh if you haven&#x27;t already, then start it up <code>$ sudo docker-compose up -d</code>. If there are local images, they&#x27;ll be used, otherwise they&#x27;ll be pulled from Docker Hub.</p><p>After about 30 minutes, you should see flows in elasticsearch.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-versioned-docs"></a>Make Versioned Docs<a class="hash-link" href="#make-versioned-docs" title="Direct link to heading">#</a></h2><p>A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the <strong>Docusaurus guide</strong>. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-new-nfdump-collector-images"></a>To Make New Nfdump-Collector Images<a class="hash-link" href="#to-make-new-nfdump-collector-images" title="Direct link to heading">#</a></h2><p>If a new version of nfdump has been released that we need, new nfdump-collector images need to be made.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git clone https://github.com/netsage-project/docker-nfdump-collector.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd docker-nfdump-collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use squash: create a file at /etc/docker/daemon.json and put into it</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;experimental&quot;: true  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;debug: false&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To push to Docker Hub and quit docker</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker push netsage/nfdump-collector:$VER</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">...</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-logstash"></a>New Version of Logstash<a class="hash-link" href="#new-version-of-logstash" title="Direct link to heading">#</a></h3><p>If a new version of logstash has been released that we want everyone to use,
-???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e4c61278.03e947b6.js"></script>
+???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/6e3166bc.fa40017d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/devel/docusaurus/index.html b/docs/devel/docusaurus/index.html
index e1861250..75f07b93 100644
--- a/docs/devel/docusaurus/index.html
+++ b/docs/devel/docusaurus/index.html
@@ -5,21 +5,21 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Revising Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Revising Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation uses Docusaurus."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation uses Docusaurus."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/0d6d53ac.2ae3b199.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Revising Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Revising Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation uses Docusaurus."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation uses Docusaurus."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d74c151a.f23bb09f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
 at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><p><em>THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-not-using-docker"></a>If Not Using Docker<a class="hash-link" href="#if-not-using-docker" title="Direct link to heading">#</a></h2><p>These are instructions for editing and releasing docs without using Docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><p>To get started the first time, install npm, then use that to install yarn </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo yum install npm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo npm install -g yarn </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd netsage-pipeline/website</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn install</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-local-development"></a>If Local Development<a class="hash-link" href="#if-local-development" title="Direct link to heading">#</a></h3><p>If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the build directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">go to http://localhost:3000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-changes"></a>To Make Changes<a class="hash-link" href="#to-make-changes" title="Direct link to heading">#</a></h3><p>Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><p>To view the changes you&#x27;ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the &quot;Deploying Docs to github.io&quot; section below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn run docusaurus docs:version a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
@@ -28,16 +28,16 @@
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release-1"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release-1" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build build_docs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
 This will create new versioned docs in website/versioned_docs/.</p><p>Then edit docusaurus.config.js and change <code>lastVersion:</code> to refer to the new version number. </p><p>Finally, commit and push the following to github:</p><ul><li>website/versioned_docs/version-a.b.c/</li><li>website/versioned_sidebars/version-a.b.c.sidebars.json</li><li>versions.json </li><li>docusaurus.config.js</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="deploying-docs-to-githubio-1"></a>Deploying Docs to github.io<a class="hash-link" href="#deploying-docs-to-githubio-1" title="Direct link to heading">#</a></h3><p>How to do this when using Docker ??? Get into the container ???</p><p>For now, go a linux server that has yarn installed and
-follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/0d6d53ac.2ae3b199.js"></script>
+follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d74c151a.f23bb09f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/components/docker_env/index.html b/docs/next/components/docker_env/index.html
index 753b4b2f..051a0d55 100644
--- a/docs/next/components/docker_env/index.html
+++ b/docs/next/components/docker_env/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_env | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="docker_env | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Next, copy env.example to .env"><meta data-react-helmet="true" property="og:description" content="Next, copy env.example to .env"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/64269ed8.20920114.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/64269ed8.18f9818d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
-For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/64269ed8.20920114.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_env">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_env">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_env">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_env">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_env">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_env">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_env">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_env">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_env</h1></header><div class="markdown"><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
+For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_env.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/64269ed8.18f9818d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/components/docker_first_steps/index.html b/docs/next/components/docker_first_steps/index.html
index 0093e012..25b9d5ad 100644
--- a/docs/next/components/docker_first_steps/index.html
+++ b/docs/next/components/docker_first_steps/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_first_steps | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="docker_first_steps | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="saving this for now in case I need to put it back #"><meta data-react-helmet="true" property="og:description" content="saving this for now in case I need to put it back #"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/080a21b4.34495c8a.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/080a21b4.616746c2.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_first_steps">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_first_steps">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_first_steps">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_first_steps">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_first_steps">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_first_steps">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_first_steps">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_first_steps</h1></header><div class="markdown"><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="saving-this-for-now-in-case-i-need-to-put-it-back"></a>saving this for now in case I need to put it back<a class="hash-link" href="#saving-this-for-now-in-case-i-need-to-put-it-back" title="Direct link to heading">#</a></h4><p>Then checkout the latest version of the code.  If you are a developer you&#x27;ll want the latest version from master, otherwise please use make sure
 you&#x27;ve checked out the latest tagged version.</p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Normal Deployment, eg, checkout version 1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git checkout v1.2.8 -b v1.2.8</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">## Developers</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git fetch </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git reset --hard origin/master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes.  On initial installation, you should not have any, but if you do wish to save any state, please make sure you commit and backup to a feature branch before continuing</p><p>Example:
-<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/080a21b4.34495c8a.js"></script>
+<code>git commit -a -m &quot;Saving local state&quot;; git checkout -b feature/backup; git checkout master</code></p></div></div><p>All instructions that follow assume these first steps were performed succesfully.  If not, you&#x27;ll likely run into errors down the line if the code doesn&#x27;t match up with the instructions provided.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_first_steps.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/080a21b4.616746c2.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/components/docker_pipeline/index.html b/docs/next/components/docker_pipeline/index.html
index f9ffbd17..0e27250d 100644
--- a/docs/next/components/docker_pipeline/index.html
+++ b/docs/next/components/docker_pipeline/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">docker_pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="docker_pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Start up the pipeline (all containers) using:"><meta data-react-helmet="true" property="og:description" content="Start up the pipeline (all containers) using:"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ba55e963.5bed7840.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ba55e963.db919392.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ba55e963.5bed7840.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/components/docker_pipeline">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/components/docker_pipeline">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/components/docker_pipeline">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/components/docker_pipeline">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/components/docker_pipeline">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/components/docker_pipeline">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/components/docker_pipeline">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/components/docker_pipeline">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">docker_pipeline</h1></header><div class="markdown"><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/components/docker_pipeline.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ba55e963.db919392.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/deploy/bare_metal_install/index.html b/docs/next/deploy/bare_metal_install/index.html
deleted file mode 100644
index 1fa92929..00000000
--- a/docs/next/deploy/bare_metal_install/index.html
+++ /dev/null
@@ -1,56 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width,initial-scale=1">
-<meta name="generator" content="Docusaurus v2.0.0-alpha.72">
-<link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Manual Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/bare_metal_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Manual Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><meta data-react-helmet="true" property="og:description" content="This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/bare_metal_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/bare_metal_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/bare_metal_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e5ce8f83.b18dcb47.js" as="script">
-</head>
-<body>
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/bare_metal_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/bare_metal_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Manual Installation Guide</h1></header><div class="markdown"><p>This document covers installing the NetSage Flow Processing Pipeline manually on a new machine (without using Docker). Steps should be followed below in order unless you know for sure what you are doing. This document assumes a RedHat Linux environment or one of its derivatives.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-sources"></a>Data sources<a class="hash-link" href="#data-sources" title="Direct link to heading">#</a></h2><p>The Processing pipeline needs data to ingest in order to do anything. There are two types of data that can be consumed.</p><ol><li>sflow or netflow</li><li>tstat</li></ol><p>At least one of these must be set up on a sensor to provide the incoming flow data.</p><p>Sflow and netflow data should be sent to ports on the pipeline host where nfcapd and/or sfcapd are ready to receive it.</p><p>Tstat data should be sent directly to the logstash input RabbitMQ queue (the same one that the Importer writes to, if it is used). From there, the data will be processed the same as sflow/netflow data.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-prerequisites"></a>Installing the Prerequisites<a class="hash-link" href="#installing-the-prerequisites" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-nfdump"></a>Installing nfdump<a class="hash-link" href="#installing-nfdump" title="Direct link to heading">#</a></h3><p>The nfdump package provides nfcapd and sfcapd processes which recieve flow data and write nfcapd files.
-The Importer also uses nfdump. If you are only collecting tstat data, you do not need nfdump. </p><p>Nfdump is <em>not</em> listed as a dependency of the Pipeline RPM package, as in a lot cases people are running special builds of nfdump -- but make sure you install it before you try running the Netflow Importer. If in doubt, <code>yum install nfdump</code> should work.
-Flow data exported by some routers require a newer version of nfdump than the one in the CentOS repos; in these cases, it may be necessary to manually compile and install the lastest nfdump. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>It is recommended to check the version of nfdump used in the Docker installation and use the same or newer in order to be sure that any fixes for impactful issues are included.</p></div></div><p>If desired, you can also install nfsen, which has a UI for viewing flow data and can manage starting and stopping all the nfcapd/sfcapd processes for you.The nfsen.conf file has a section in which to configure all the sources. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-rabbitmq"></a>Installing RabbitMQ<a class="hash-link" href="#installing-rabbitmq" title="Direct link to heading">#</a></h3><p>The pipeline requires a RabbitMQ server. Typically, this runs on the same server as the pipeline itself, but if need be, you can separate them (for this reason, the Rabbit server is not automatically installed with the pipeline package).</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install rabbitmq-server</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Typically, the default configuration will work. Perform any desired Rabbit configuration, then, start RabbitMQ:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service rabbitmq-server start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start rabbitmq-server.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-logstash"></a>Installing Logstash<a class="hash-link" href="#installing-logstash" title="Direct link to heading">#</a></h3><p>See the logstash documentation. We are currently using Version 7.10.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-epel-repo"></a>Installing the EPEL repo<a class="hash-link" href="#installing-the-epel-repo" title="Direct link to heading">#</a></h3><p>Some of our dependencies come from the EPEL repo. To install this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install epel-release</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-globalnoc-open-source-repo"></a>Installing the GlobalNOC Open Source repo<a class="hash-link" href="#installing-the-globalnoc-open-source-repo" title="Direct link to heading">#</a></h3><p>The Pipeline package (and its dependencies that are not in EPEL) are in the GlobalNOC Open Source Repo.</p><p>For Red Hat/CentOS 6, create <code>/etc/yum.repos.d/grnoc6.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc6]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el6 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/6/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC6</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>For Red Hat/CentOS 7, create <code>/etc/yum.repos.d/grnoc7.repo</code> with the following content.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[grnoc7]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">name=GlobalNOC Public el7 Packages - $basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">baseurl=https://repo-public.grnoc.iu.edu/repo/7/$basearch</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">enabled=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgcheck=1</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">gpgkey=https://repo-public.grnoc.iu.edu/repo/RPM-GPG-KEY-GRNOC7</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The first time you install packages from the repo, you will have to accept the GlobalNOC repo key.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing-the-pipeline-importer-and-logstash-configs"></a>Installing the Pipeline (Importer and Logstash configs)<a class="hash-link" href="#installing-the-pipeline-importer-and-logstash-configs" title="Direct link to heading">#</a></h2><p>Install it like this:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# yum install grnoc-netsage-deidentifier</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Pipeline components:</p><ol><li>Flow Filter - GlobalNOC uses this for Cenic data to filter out some flows. Not needed otherwise.</li><li>Netsage Netflow Importer - required to read nfcapd files from sflow and netflow importers. (If using tstat flow sensors only, this is not needed.)</li><li>Logstash - be sure the number of logstash pipeline workers in /etc/logstash/logstash.yml is set to 1 or flow stitching/aggregation will not work right!  </li><li>Logstash configs - these are executed in alphabetical order.  See the Logstash doc. At a minimum, the input, output, and aggregation configs have parameters that you will need to update or confirm.</li></ol><p>Nothing will automatically start after installation as we need to move on to configuration. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer-configuration"></a>Importer Configuration<a class="hash-link" href="#importer-configuration" title="Direct link to heading">#</a></h2><p>Configuration files of interest are</p><ul><li>/etc/grnoc/netsage/deidentifier/netsage_shared.xml - Shared config file allowing configuration of collections, and Rabbit connection information</li><li>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml - other settings</li><li>/etc/grnoc/netsage/deidentifier/logging.conf - logging config</li><li>/etc/grnoc/netsage/deidentifier/logging-debug.conf - logging config with debug enabled</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-shared-config-file"></a>Setting up the shared config file<a class="hash-link" href="#setting-up-the-shared-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_shared.xml</code></p><p>There used to be many perl-based pipeline components and daemons. At this point, only the importer is left, the rest having been replaced by logstash.  The shared config file, which was formerly used by all the perl components, is read before reading the individual importer config file.</p><p>The most important part of the shared configuration file is the definition of collections. Each sflow or netflow sensor will have its own collection stanza. Here is one such stanza, a netflow example. Instance and router-address can be left commented out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;collection&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Top level directory of the nfcapd files for this sensor (within this dir are normally year directories, etc.) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-path&gt;/path/to/netflow-files/&lt;/flow-path&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Sensor name - can be the hostname or any string you like. Shows up in grafana dashboards.  --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;sensor&gt;Netflow Sensor 1&lt;/sensor&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Flow type - sflow or netflow (defaults to netflow) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">         &lt;flow-type&gt;sflow&lt;/flow-type&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &quot;instance&quot; goes along with sensor.  This is to identify various instances if a sensor has --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- more than one &quot;stream&quot; / data collection.  Defaults to 0. --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &lt;instance&gt;1&lt;/instance&gt; --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- Used in Flow-Filter. Defaults to sensor, but you can set it to something else here --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     &lt;!-- &lt;router-address&gt;&lt;/router-address&gt; --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">&lt;/collection&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Having multiple collections in one importer can sometimes cause issues for aggregation, as looping through the collections one at a time adds to the time between the flows, affecting timeouts. You can also set up multiple Importers with differently named shared and importer config files and separate init.d files. </p><p>There is also RabbitMQ connection information in the shared config, though queue names are set in the Importer config. (The Importer does not read from a rabbit queue, but other old components did, so both input and output are set.) </p><p>Ideally, flows should be deidentified before they leave the host on which the data is stored. If flows that have not be deidentified need to be pushed to another node for some reason, the Rabbit connection must be encrypted with SSL.</p><p>If you&#x27;re running a default RabbitMQ config, which is open only to &#x27;localhost&#x27; as guest/guest, you won&#x27;t need to change anything here.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;!-- rabbitmq connection info --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;rabbit_input&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;host&gt;127.0.0.1&lt;/host&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;port&gt;5672&lt;/port&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;username&gt;guest&lt;/username&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;password&gt;guest&lt;/password&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;ssl&gt;0&lt;/ssl&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;batch_size&gt;100&lt;/batch_size&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;vhost&gt;/&lt;/vhost&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;durable&gt;1&lt;/durable&gt; &lt;!-- Whether the rabbit queue is &#x27;durable&#x27; (don&#x27;t change this unless you have a reason) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/rabbit_input&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;rabbit_output&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;host&gt;127.0.0.1&lt;/host&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;port&gt;5672&lt;/port&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;username&gt;guest&lt;/username&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;password&gt;guest&lt;/password&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;ssl&gt;0&lt;/ssl&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;batch_size&gt;100&lt;/batch_size&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;vhost&gt;/&lt;/vhost&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    &lt;durable&gt;1&lt;/durable&gt; &lt;!-- Whether the rabbit queue is &#x27;durable&#x27; (don&#x27;t change this unless you have a reason) --&gt;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  &lt;/rabbit_output&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="setting-up-the-importer-config-file"></a>Setting up the Importer config file<a class="hash-link" href="#setting-up-the-importer-config-file" title="Direct link to heading">#</a></h3><p><code>/etc/grnoc/netsage/deidentifier/netsage_netflow_importer.xml</code></p><p>This file has a few more setting specific to the Importer component which you may like to adjust.  </p><ul><li>Rabbit_output has the name of the output queue. This should be the same as that of the logstash input queue.  </li><li>(The Importer does not actually use an input rabbit queue, so we add a &quot;fake&quot; one here.)</li><li>Min-bytes is a threshold applied to flows aggregated within one nfcapd file. Flows smaller than this will be discarded.</li><li>Min-file-age is used to be sure files are complete before being read. </li><li>Cull-enable and cull-ttl can be used to have nfcapd files older than some number of days automatically deleted. </li><li>Pid-file is where the pid file should be written. Be sure this matches what is used in the init.d file.</li><li>Keep num-processes set to 1.</li></ul><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--  NOTE: Values here override those in the shared config --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- rabbitmq queues --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_input</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">netsage_deidentifier_netflow_fake</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">2</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_input</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_output</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">3</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">channel</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">netsage_deidentifier_raw</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">queue</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">rabbit_output</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- How many flows to process at once --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-batch-size</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">100</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-batch-size</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- How many concurrent workers should perform the necessary operations --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">num-processes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">num-processes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- path to nfdump executable (defaults to /usr/bin/nfdump) --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--   &lt;nfdump-path&gt;/path/to/nfdump&lt;/nfdump-path&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Where to store the cache, where it tracks what files it has/hasn&#x27;t read --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cache-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/cache/netsage/netflow_importer.cache</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cache-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- The minium flow size threshold - will not  import any flows smaller than this --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Defaults to 500M  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">min-bytes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">100000000</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">min-bytes</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"> </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- Do not import nfcapd files younger than min-file-age</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        The value must match /^(\d+)([DWMYhms])$/ where D, W, M, Y, h, m and s are</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        &quot;day(s)&quot;, &quot;week(s)&quot;, &quot;month(s)&quot;, &quot;year(s)&quot;, &quot;hour(s)&quot;, &quot;minute(s)&quot; and &quot;second(s)&quot;, respectively&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        See http://search.cpan.org/~pfig/File-Find-Rule-Age-0.2/lib/File/Find/Rule/Age.pm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">        Default: 0 (no minimum age) </span></div><div class="token-line" style="color:#bfc7d5"><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">    --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">min-file-age</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">10m</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">min-file-age</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"> </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- cull-enable: whether to cull processed flow data files --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- default: no culling; set to 1 to turn culling on --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--    &lt;cull-enable&gt;1&lt;/cull-enable&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- cull-tty: cull time to live, in days --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- number of days to retain imported data files before deleting them; default: 3 --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!--    &lt;cull-ttl&gt;5&lt;/cull-ttl&gt;  --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token comment" style="color:rgb(105, 112, 152);font-style:italic">&lt;!-- where should we write the daemon pid file to --&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/var/run/netsage-netflow-importer-daemon.pid</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">pid-file</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">master</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"></span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">config</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-setup-notes"></a>Logstash Setup Notes<a class="hash-link" href="#logstash-setup-notes" title="Direct link to heading">#</a></h2><p>Standard logstash filter config files are provided with this package. Most should be used as-is, but the input and output configs may be modified for your use.</p><p>The aggregation filter also has settings that may be changed as well - check the two timeouts and the aggregation maps path. </p><p>When upgrading, these logstash configs will not be overwritten. Be sure any changes get copied into the production configs.</p><p>FOR FLOW STITCHING/AGGREGATION - IMPORTANT!
-Flow stitching (ie, aggregation) will NOT work properly with more than ONE logstash pipeline worker!
-Be sure to set &quot;pipeline.workers: 1&quot; in /etc/logstash/logstash.yml and/or /etc/logstash/pipelines.yml. When running logstash on the command line, use &quot;-w 1&quot;.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-logstash"></a>Start Logstash<a class="hash-link" href="#start-logstash" title="Direct link to heading">#</a></h2><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service logstash start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start logstash.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>It will take couple minutes to start. Log files are normally /var/log/messages and /var/log/logstash/logstash-plain.log.</p><p>When logstash is stopped, any flows currently &quot;in the aggregator&quot; will be written out to /tmp/logstash-aggregation-maps (or the path/file set in 40-aggregation.conf). These will be read in and deleted when logstash is started again. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-importer"></a>Start the Importer<a class="hash-link" href="#start-the-importer" title="Direct link to heading">#</a></h2><p>Typically, the daemons are started and stopped via init script (CentOS 6) or systemd (CentOS 7). They can also be run manually. The daemons all support these flags:</p><p><code>--config [file]</code> - specify which config file to read</p><p><code>--sharedconfig [file]</code> - specify which shared config file to read</p><p><code>--logging [file]</code> - the logging config</p><p><code>--nofork</code> - run in foreground (do not daemonize)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">[root@host ~]# /sbin/service netsage-netflow-importer start </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">          or # systemctl start netsage-netflow-importer.service</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The Importer will create a deamon process and a worker process. When stopping the service, the worker process might take a few minutes to quit. If it does not quit, kill it by hand. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="cron-jobs"></a>Cron jobs<a class="hash-link" href="#cron-jobs" title="Direct link to heading">#</a></h2><p>Sample cron files are provided. Please review and uncomment their contents. These periodically download MaxMind, CAIDA, and Science Registry files, and also restart logstash. Logstash needs to be restarted in order for any updated files to be read in. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/bare_metal_install.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/choose_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Choosing an Installation Procedure</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-sources" class="table-of-contents__link">Data sources</a></li><li><a href="#installing-the-prerequisites" class="table-of-contents__link">Installing the Prerequisites</a><ul><li><a href="#installing-nfdump" class="table-of-contents__link">Installing nfdump</a></li><li><a href="#installing-rabbitmq" class="table-of-contents__link">Installing RabbitMQ</a></li><li><a href="#installing-logstash" class="table-of-contents__link">Installing Logstash</a></li><li><a href="#installing-the-epel-repo" class="table-of-contents__link">Installing the EPEL repo</a></li><li><a href="#installing-the-globalnoc-open-source-repo" class="table-of-contents__link">Installing the GlobalNOC Open Source repo</a></li></ul></li><li><a href="#installing-the-pipeline-importer-and-logstash-configs" class="table-of-contents__link">Installing the Pipeline (Importer and Logstash configs)</a></li><li><a href="#importer-configuration" class="table-of-contents__link">Importer Configuration</a><ul><li><a href="#setting-up-the-shared-config-file" class="table-of-contents__link">Setting up the shared config file</a></li><li><a href="#setting-up-the-importer-config-file" class="table-of-contents__link">Setting up the Importer config file</a></li></ul></li><li><a href="#logstash-setup-notes" class="table-of-contents__link">Logstash Setup Notes</a></li><li><a href="#start-logstash" class="table-of-contents__link">Start Logstash</a></li><li><a href="#start-the-importer" class="table-of-contents__link">Start the Importer</a></li><li><a href="#cron-jobs" class="table-of-contents__link">Cron jobs</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e5ce8f83.b18dcb47.js"></script>
-</body>
-</html>
\ No newline at end of file
diff --git a/docs/next/deploy/choose_install/index.html b/docs/next/deploy/choose_install/index.html
deleted file mode 100644
index db85aada..00000000
--- a/docs/next/deploy/choose_install/index.html
+++ /dev/null
@@ -1,34 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width,initial-scale=1">
-<meta name="generator" content="Docusaurus v2.0.0-alpha.72">
-<link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Choosing an Installation Procedure | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/choose_install"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Choosing an Installation Procedure | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Manual or BareMetal Installation"><meta data-react-helmet="true" property="og:description" content="Manual or BareMetal Installation"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/choose_install"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/choose_install" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/choose_install" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/66e31e6e.ce62be57.js" as="script">
-</head>
-<body>
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/choose_install">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/choose_install">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/choose_install">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/choose_install">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/choose_install">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/choose_install">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/choose_install">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/deploy/choose_install">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/deploy/choose_install">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/choose_install">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Choosing an Installation Procedure</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="manual-or-baremetal-installation"></a>Manual or BareMetal Installation<a class="hash-link" href="#manual-or-baremetal-installation" title="Direct link to heading">#</a></h2><p>The Manual (baremetal) Installation Guide will walk you through installing the pipeline using your own server infrastructure and requires you to maintain all the components involved.</p><p>It will likely be a bit better when it comes to performance, and have greater flexibility, but there is also more complexity involved in configuring and setting up.</p><p>If you are the ultimate consumer of the data then setting up a baremetal version might be worth doing. Or at least the final rabbitMQ that will be holding the data since it&#x27;ll like need to handle a large dataset.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="dockerized-version"></a>Dockerized Version<a class="hash-link" href="#dockerized-version" title="Direct link to heading">#</a></h2><p>The Docker version makes it trivial to bring up the pipeline for both a developer and consumer. The work is mostly already done for you. It should be a simple matter of configuring a few env settings and everything should &#x27;just&#x27; work.</p><p>If you are simply using the pipeline to deliver the anonymized network stats for someone else&#x27;s consumption, then using the docker pipeline would be preferred.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="choose-your-adventure"></a>Choose your adventure<a class="hash-link" href="#choose-your-adventure" title="Direct link to heading">#</a></h2><ul><li><a href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual/Server Installation</a></li><li><a href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Simple Docker</a> - 1 netflow sensor and/or 1 sflow sensor</li><li><a href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Advanced Docker</a> - options that allow for more complex configurations</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/choosing.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Manual Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#manual-or-baremetal-installation" class="table-of-contents__link">Manual or BareMetal Installation</a></li><li><a href="#dockerized-version" class="table-of-contents__link">Dockerized Version</a></li><li><a href="#choose-your-adventure" class="table-of-contents__link">Choose your adventure</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/66e31e6e.ce62be57.js"></script>
-</body>
-</html>
\ No newline at end of file
diff --git a/docs/next/deploy/docker_install_advanced/index.html b/docs/next/deploy/docker_install_advanced/index.html
index 5b6ce39d..31827072 100644
--- a/docs/next/deploy/docker_install_advanced/index.html
+++ b/docs/next/deploy/docker_install_advanced/index.html
@@ -6,34 +6,33 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Advanced Options Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Docker Advanced Options Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><meta data-react-helmet="true" property="og:description" content="If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/cbd1c0d0.5208c323.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/cbd1c0d0.6ea2feaf.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_advanced">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_advanced">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Advanced Options Guide</h1></header><div class="markdown"><p>If the basic Docker Installation does not meet your needs, the following customizations will allow for more complex situations. Find the section(s) which apply to you.</p><p><em>Please first read the Docker Installation guide in detail. This guide will build on top of that.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-add-an-additional-sflow-or-netflow-collector"></a>To Add an Additional Sflow or Netflow Collector<a class="hash-link" href="#to-add-an-additional-sflow-or-netflow-collector" title="Direct link to heading">#</a></h2><p>If you have more than 1 sflow and/or 1 netflow sensor, you will need to create more collectors and modify the importer config file. The following instructions describe the steps needed to add one additional sensor.</p><p>Any number of sensors can be accomodated, although if there are more than a few being processed by the same Importer, you may run into issues where long-lasting flows from sensosr A time out in the aggregation step while waiting for flows from sensors B to D to be processed. (Another option might be be to run more than one Docker deployment.) </p><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="a-edit-docker-composeoverrideyml"></a>a. Edit docker-compose.override.yml<a class="hash-link" href="#a-edit-docker-composeoverrideyml" title="Direct link to heading">#</a></h4><p>The pattern to add a flow collector is always the same. To add an sflow collector called example-collector, edit the docker-compose.override.yml file and add something like</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">example-collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/nfdump</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">collector</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">alpine</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">restart</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> always</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">command</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> sfcapd </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">T all </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">l /data </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">S 1 </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">w </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">z </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">p 9997</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./data/input_data/example</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token key atrule">ports</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> </span><span class="token string" style="color:rgb(195, 232, 141)">&quot;9997:9997/udp&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>collector name: should be updated to something that has some meaning, in our example &quot;example-collector&quot;.</li><li>image: copy from the default collector sections already in the file. </li><li>command: choose between &quot;sfcapd&quot; for sflow and &quot;nfcapd&quot; for netflow, and at the end of the command, specify the port to watch for incoming flow data.  </li><li>volumes: specify where to write the nfcapd files. Make sure the path is unique and in ./data/. In this case, we&#x27;re writing to ./data/input_data/example. Change &quot;example&quot; to something meaningful.</li><li>ports: make sure the port here matches the port you&#x27;ve set in the command. Naturally all ports have to be unique for this host and the router should be configured to export data to the same port. (?? If the port on your docker container is different than the port on your host/local machine, use container_port:host_port.) </li></ul><p>Make sure the indentation is right or you&#x27;ll get an error about yaml parsing.</p><p>You will also need to uncomment these lines: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="b--edit-netsage_overridexml"></a>b.  Edit netsage_override.xml<a class="hash-link" href="#b--edit-netsage_overridexml" title="Direct link to heading">#</a></h4><p>To make the Pipeline Importer aware of the new data to process, you will need to create a custom Importer configuration: netsage_override.xml.  This will replace the usual config file netsage_shared.xml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Edit netsage_override.xml and add a new &quot;collection&quot; section for the new sensor as in the following example. The flow-path should match the path set above in docker-compose.override.yml. $exampleSensorName is a new &quot;variable&quot;; don&#x27;t replace it here, it will be replaced with a value that you set in the .env file. For the flow-type, enter &quot;sflow&quot; or &quot;netflow&quot; as appropriate. (Enter &quot;netflow&quot; if you&#x27;re running IPFIX.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">/data/input_data/example/</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-path</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">$exampleSensorName</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">sensor</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">        </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">sflow</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">flow-type</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">collection</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="c-edit-environment-file"></a>c. Edit environment file<a class="hash-link" href="#c-edit-environment-file" title="Direct link to heading">#</a></h4><p>Then, in the .env file, add a line that sets a value for the &quot;variable&quot; you referenced above, $exampleSensorName. The value is the name of the sensor which will be saved to elasticsearch and which appears in Netsage Dashboards. Set it to something meaningful and unique. E.g.,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly ini"><div tabindex="0" class="prism-code language-ini codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">exampleSensorName=MyNet Los Angeles sFlow</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h4><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="d-running-the-new-collector"></a>d. Running the new collector<a class="hash-link" href="#d-running-the-new-collector" title="Direct link to heading">#</a></h4><p>After doing the setup above and selecting the docker version to run, you can start the new collector by running the following line, using the collector name (or by running <code>docker-compose up -d</code> to start up all containers):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d example-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-keep-only-flows-from-certain-interfaces"></a>To Keep Only Flows From Certain Interfaces<a class="hash-link" href="#to-keep-only-flows-from-certain-interfaces" title="Direct link to heading">#</a></h2><p>If your sensors are exporting all flows, but only those using a particular interface are relevant, use this option in the .env file. The collectors and importer will save/read all incoming flows, but the logstash pipeline will drop those that do not have src_ifindex OR dst_inindex equal to one of those listed. </p><p>In the .env file, uncomment lines in the appropriate section and enter the information required. Be sure <code>ifindex_filter_flag=True</code> with &quot;True&quot; capitalized as shown, any sensor names are spelled exactly right, and list all the ifindex values of flows that should be kept and processed. Some examples (use just one!):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123,456</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=Sensor 1: 789</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_filter_keep=123; Sensor 1: 789; Sensor 2: 800, 900</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In the first case, all flows that have src_ifindex = 123 or dst_ifindex = 123 will be kept, regardless of sensor name. (Note that this may be a problem if you have more than 1 sensor with the same ifindex values!)
 In the 2nd case, if src or dst ifindex is 123 or 456, the flow will be processed.
 In the 3rd case, only flows from Sensor 1 will be filtered, with flows using ifindex 789 kept.
-In the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  </p><p>Spaces don&#x27;t matter except within the sensor names. Punctuation is required as shown.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-filter-flows-by-subnet"></a>To Filter Flows by Subnet<a class="hash-link" href="#to-filter-flows-by-subnet" title="Direct link to heading">#</a></h2><p>With this option, flows from specified sensors will be dropped unless src or dst is in the list of subnets to keep. It works similarly to the option to filter by interface.  &quot;ALL&quot; can refer to all sensors.
-If a sensor is not referenced at all, all of its flows will be kept. </p><p>For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">subnet_filter_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">subnet_filter_keep=Sensor A Name: 123.45.6.0/16; Sensor B Name: 123.33.33.0/24, 456.66.66.0/24</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk doesn&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-save-flow-data-to-a-different-location"></a>To Save Flow Data to a Different Location<a class="hash-link" href="#to-save-flow-data-to-a-different-location" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options.</p><ol><li>The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.</li></ol><p>During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline dir}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir /var/netsage</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rm data/.placeholder</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rmdir data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ln -s /var/netsage {netsage-pipeline dir}/data</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(Check the permissions of the directory.)</p><ol start="2"><li>Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to <code>- /mydir/input_data/netflow:/data</code> (similarly for sflow) and set the importer and logstash volumes to <code>- /mydir:/data</code>. </li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.
-You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/next/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-filter-flows-by-subnet" class="table-of-contents__link">To Filter Flows by Subnet</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/cbd1c0d0.5208c323.js"></script>
+In the last example, any flow with ifindex 123 will be kept. Sensor 1 flows with ifindex 789 (or 123) will be kept, and those from Sensor 2 having ifindex 800 or 900 (or 123) will be kept.  </p><p>Spaces don&#x27;t matter except within the sensor names. Punctuation is required as shown.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-a-sensor-name-depending-on-the-interface-used"></a>To Change a Sensor Name Depending on the Interface Used<a class="hash-link" href="#to-change-a-sensor-name-depending-on-the-interface-used" title="Direct link to heading">#</a></h2><p>In some cases, users want to keep all flows from a certain sensor but differentiate between those that enter or exit through specific sensor interfaces. This can be done by using this option in the .env file.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 4 fields are set properly! For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_old_name=IU Sflow </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_new_name=IU Bloomington Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ifindex_sensor_rename_ifindex=10032</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>In this case, any flows from the &quot;IU Sflow&quot; sensor that use interface 10032 (src_ifindex = 10032 OR dst_ifindex = 10032) will have the sensor name changed from &quot;IU Sflow&quot; to &quot;IU Bloomington Sflow&quot;. Currently, only one such rename can be configured in Docker and only 1 ifindex is allowed.</p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>Please notify the devs at IU in advance, if you need to modify a sensor name, because the regexes used for determining sensor_group and sensor_type may have to be updated.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-do-sampling-rate-corrections-in-logstash"></a>To Do Sampling Rate Corrections in Logstash<a class="hash-link" href="#to-do-sampling-rate-corrections-in-logstash" title="Direct link to heading">#</a></h2><p>When flow sampling is done, corrections have to be applied. For example, if you are sampling 1 out of 100 flows, for each flow measured, it is assumed that in reality there would be 100 flows of that size with that src and dst, so the number of bits (and the number of packets, bits/s and packets/s) is multiplied by 100. Usually the collector (nfcapd or sfcapd process) gets the sampling rate from the incoming data and applies the correction, but in some cases, the sensor may not send the sampling rate, or there may be a complex set-up that requires a manual correction. With netflow, a manual correction can be applied using the &#x27;-s&#x27; option in the nfsen config, if nfsen is being used, or the nfcapd command, but this is not convenient when using Docker. For sflow, there is no such option. In either case, the correction can be made in logstash as follows.</p><p>In the .env file, uncomment the appropriate section and enter the information required. Be sure &quot;True&quot; is capitalized as shown and all 3 fields are set properly! The same correction can be applied to multiple sensors by using a comma-separed list. The same correction applies to all listed sensors. For example,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_flag=True</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_sensors=IU Bloomington Sflow, IU Sflow</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sampling_correction_factor=512</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-change-how-long-nfcapd-files-are-kept"></a>To Change How Long Nfcapd Files Are Kept<a class="hash-link" href="#to-change-how-long-nfcapd-files-are-kept" title="Direct link to heading">#</a></h2><p>The importer will automatically delete older nfcapd files for you, so that your disk doesn&#x27;t fill up. By default, 3 days worth of files will be kept. This can be adjusted by making a netsage_override.xml file:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp compose/importer/netsage_shared.xml userConfig/netsage_override.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>At the bottom of the file, edit this section to set the number of days worth of files to keep. Set cull-enable to 0 for no culling. Eg, to save 1 days worth of data:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly xml"><div tabindex="0" class="prism-code language-xml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-enable</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain">1</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">cull-ttl</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&lt;/</span><span class="token tag" style="color:rgb(255, 85, 114)">worker</span><span class="token tag punctuation" style="color:rgb(199, 146, 234)">&gt;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>You will also need to uncomment these lines in docker-compose.override.yml: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/netsage_override.xml</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/etc/grnoc/netsage/deidentifier/netsage_shared.xml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-save-flow-data-to-a-different-location"></a>To Save Flow Data to a Different Location<a class="hash-link" href="#to-save-flow-data-to-a-different-location" title="Direct link to heading">#</a></h2><p>By default, data is saved to subdirectories in the ./data/ directory (ie, the data/ directory in the git checkout).  If you would like to use a different location, there are two options.</p><ol><li>The best solution is to create a symlink between ./data/ and the preferred location, or, for an NFS volume, export it as ${PROJECT_DIR}/data.</li></ol><p>During installation, delete the data/ directory (it should only contain .placeholder), then create your symlink. Eg, to use /var/netsage/ instead of data/, </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline dir}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">mkdir /var/netsage</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rm data/.placeholder</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rmdir data</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">ln -s /var/netsage {netsage-pipeline dir}/data</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>(Check the permissions of the directory.)</p><ol start="2"><li>Alternatively, update volumes in docker-compose.yml and docker-compose.override.yml Eg, to save nfcapd files to subdirs in /mydir, set the collector volumes to <code>- /mydir/input_data/netflow:/data</code> (similarly for sflow) and set the importer and logstash volumes to <code>- /mydir:/data</code>. </li></ol><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>If you choose to update the docker-compose file, keep in mind that those changes will cause a merge conflict or be wiped out on upgrade.
+You&#x27;ll have to manage the volumes exported and ensure all the paths are updated correctly for the next release manually.</p></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-customize-java-settings--increase-memory-available-for-lostash"></a>To Customize Java Settings / Increase Memory Available for Lostash<a class="hash-link" href="#to-customize-java-settings--increase-memory-available-for-lostash" title="Direct link to heading">#</a></h2><p>If cpu or memory seems to be a problem, try increasing the JVM heap size for logstash from 2GB to 3 or 4, no more than 8.</p><p>To do this, edit LS_JAVA_OPTS in the .env file. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">LS_JAVA_OPTS=</span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xmx4g </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain">Xms4g</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Here are some tips for adjusting the JVM heap size (<a href="https://www.elastic.co/guide/en/logstash/current/jvm-settings.html" target="_blank" rel="noopener noreferrer">https://www.elastic.co/guide/en/logstash/current/jvm-settings.html</a>):</p><ul><li>Set the minimum (Xms) and maximum (Xmx) heap allocation size to the same value to prevent the heap from resizing at runtime, which is a very costly process.</li><li>CPU utilization can increase unnecessarily if the heap size is too low, resulting in the JVM constantly garbage collecting. You can check for this issue by doubling the heap size to see if performance improves.</li><li>Do not increase the heap size past the amount of physical memory. Some memory must be left to run the OS and other processes. As a general guideline for most installations, don’t exceed 50-75% of physical memory. The more memory you have, the higher percentage you can use.</li></ul><p>To modify other logstash settings, rename the provided example file for JVM Options and tweak the settings as desired:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp userConfig/jvm.options_example userConfig/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Also update the docker-compose.override.xml file to uncomment lines in the logstash section. It should look something like this: </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly yaml"><div tabindex="0" class="prism-code language-yaml codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token key atrule">logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">image</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"> netsage/pipeline_logstash</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">latest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">  </span><span class="token key atrule">volumes</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain"></span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(199, 146, 234)">-</span><span class="token plain"> ./userConfig/jvm.options</span><span class="token punctuation" style="color:rgb(199, 146, 234)">:</span><span class="token plain">/usr/share/logstash/config/jvm.options</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-bring-up-kibana-and-elasticsearch-containers"></a>To Bring up Kibana and Elasticsearch Containers<a class="hash-link" href="#to-bring-up-kibana-and-elasticsearch-containers" title="Direct link to heading">#</a></h2><p>The file docker-compose.develop.yaml can be used in conjunction with docker-compose.yaml to bring up the optional Kibana and Elastic Search components.</p><p>This isn&#x27;t a production pattern but the tools can be useful at times. Please refer to the <a href="/netsage-pipeline/docs/next/devel/docker_dev_guide#optional-elasticsearch-and-kibana">Docker Dev Guide</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_advanced.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Upgrading »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#to-add-an-additional-sflow-or-netflow-collector" class="table-of-contents__link">To Add an Additional Sflow or Netflow Collector</a></li><li><a href="#to-keep-only-flows-from-certain-interfaces" class="table-of-contents__link">To Keep Only Flows From Certain Interfaces</a></li><li><a href="#to-change-a-sensor-name-depending-on-the-interface-used" class="table-of-contents__link">To Change a Sensor Name Depending on the Interface Used</a></li><li><a href="#to-do-sampling-rate-corrections-in-logstash" class="table-of-contents__link">To Do Sampling Rate Corrections in Logstash</a></li><li><a href="#to-change-how-long-nfcapd-files-are-kept" class="table-of-contents__link">To Change How Long Nfcapd Files Are Kept</a></li><li><a href="#to-save-flow-data-to-a-different-location" class="table-of-contents__link">To Save Flow Data to a Different Location</a></li><li><a href="#to-customize-java-settings--increase-memory-available-for-lostash" class="table-of-contents__link">To Customize Java Settings / Increase Memory Available for Lostash</a></li><li><a href="#to-bring-up-kibana-and-elasticsearch-containers" class="table-of-contents__link">To Bring up Kibana and Elasticsearch Containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/cbd1c0d0.6ea2feaf.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/deploy/docker_install_simple/index.html b/docs/next/deploy/docker_install_simple/index.html
index 3d26f1b8..650549ed 100644
--- a/docs/next/deploy/docker_install_simple/index.html
+++ b/docs/next/deploy/docker_install_simple/index.html
@@ -6,36 +6,36 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Installation Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Docker Installation Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><meta data-react-helmet="true" property="og:description" content="In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/734dc415.80010e23.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/734dc415.9fadde42.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_install_simple">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_install_simple">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Installation Guide</h1></header><div class="markdown"><p>In this deployment guide, you will learn how to deploy a basic Netsage setup that includes one sflow and/or one netflow collector.  If you have more than one collector of either type, or other special situations, see the Docker Advanced guide.</p><p>The Docker containers included in the installation are</p><ul><li>rabbit    (the local RabbitMQ server)</li><li>sflow-collector   (receives sflow data and writes nfcapd files)</li><li>netflow-collector   (receives netflow data and writes nfcapd files)</li><li>importer   (reads nfcapd files and puts flows into a local rabbit queue)</li><li>logstash   (logstash pipeline that processes flows and sends them to their final destination, by default a local rabbit queue)</li><li>ofelia   (cron-like downloading of files used by the logstash pipeline)</li></ul><p>The code and configs for the importer and logstash pipeline can be viewed in the netsage-project/netsage-pipeline github repo. See netsage-project/docker-nfdump-collector for code related to the collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="1-set-up-data-sources"></a>1. Set up Data Sources<a class="hash-link" href="#1-set-up-data-sources" title="Direct link to heading">#</a></h3><p>The data processing pipeline needs data to ingest in order to do anything, of course. There are three types of data that can be consumed.</p><ul><li>sflow </li><li>netflow</li><li>tstat</li></ul><p>At least one of these must be set up on a <em>sensor</em> (i.e., flow <em>exporter</em> / router), to provide the incoming flow data.
 You can do this step later, but it will helpful to have it working first. </p><p>Sflow and netflow data should be exported to the pipeline host where there will be <em>collectors</em> (nfcapd and/or sfcapd processes) ready to receive it (see below). To use the default settings, send sflow to port 9998 and netflow/IPFIX to port 9999. On the pipeline host, allow incoming traffic from the flow exporters, of course.</p><p>Tstat data should be sent directly to the logstash input rabbit queue &quot;netsage_deidentifier_raw&quot; on the pipeline host. No collector is needed for tstat data. See the netsage-project/tstat-transport repo.  (From there, logstash will grab the data and process it the same way as it processes sflow/netflow data. (See the Docker Advanced guide.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="2-set-up-a-pipeline-host"></a>2. Set up a Pipeline Host<a class="hash-link" href="#2-set-up-a-pipeline-host" title="Direct link to heading">#</a></h3><p>Decide where to run the Docker Pipeline and get it set up. Adjust iptables to allow the flow exporters (routers) to send flow data to the host. </p><p>Install Docker Engine (docker-ce, docker-ce-cli, containerd.io) - see instructions at <a href="https://docs.docker.com/engine/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/engine/install/</a>.</p><p>Install Docker Compose from Docker&#x27;s GitHub repository - see <a href="https://docs.docker.com/compose/install/" target="_blank" rel="noopener noreferrer">https://docs.docker.com/compose/install/</a>.  You need to <strong>specify version 1.29.2</strong> (or newer) in the curl command. </p><p>Check default file permissions. If the <em>logstash</em> user is not able to access the logstash config files in the git checkout, you&#x27;ll get an error from logstash saying there are no .conf files found even though they are there. Various components also need to be able to read and write to the data/ directory in the checkout. Defaults of 775 (u=rwx, g=rwx, o=rx) should work.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="3-clone-the-netsage-pipeline-project"></a>3. Clone the Netsage Pipeline Project<a class="hash-link" href="#3-clone-the-netsage-pipeline-project" title="Direct link to heading">#</a></h3><p>Clone the netsage-pipeline project from github.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When the pipeline runs, it uses the logstash conf files that are in the git checkout (in conf-logstash/), as well as a couple other files like docker-compose.yml, so it is important to checkout the correct version.</p><p>Move into the netsage-pipeline/ directory (<strong>all git and docker commands must be run from inside this directory!</strong>), then checkout the most recent version of the code. It will say you are in &#x27;detached HEAD&#x27; state if you don&#x27;t include -b.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout {tag}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Replace &quot;{tag}&quot; with the release version you intend to use, e.g., &quot;v1.2.11&quot;.  (&quot;Master&quot; is the development version and is not intended for general use!)
 <code>git status</code> will confirm which branch you are on, e.g., master or v1.2.11.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="4-create-docker-composeoverrideyml"></a>4. Create Docker-compose.override.yml<a class="hash-link" href="#4-create-docker-composeoverrideyml" title="Direct link to heading">#</a></h3><p>Information in the <code>docker-compose.yml</code> file tells docker which containers (processes) to run and sets various parameters for them.
 Settings in the <code>docker-compose.override.yml</code> file will overrule and add to those. Note that docker-compose.yml should not be edited since upgrades will replace it. Put all customizations in the override file, since override files will not be overwritten.</p><p>Collector settings may need to be edited by the user, so the information that docker uses to run the collectors is specified (only) in the override file. Therefore, docker-compose_override.example.yml must always be copied to docker-compose_override.yml. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp docker-compose.override_example.yml docker-compose.override.yml</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>By default docker will bring up a single sflow collector and a single netflow collector that listen to udp traffic on ports localhost:9998 and 9999. If this matches your case, you don&#x27;t need to make any changes to the docker-compose.override_example.yml. </p><ul><li>If you have only one collector, remove or comment out the section for the one not needed so the collector doesn&#x27;t run and simply create empty nfcapd files.</li><li>If the collectors need to listen to different ports, make the appropriate changes here in both the &quot;command:&quot; and &quot;ports:&quot; lines. </li><li>By default, the collectors will save flows to nfcapd files in sflow/ and netflow/ subdirectories in <code>./data/input_data/</code> (i.e., the data/ directory in the git checkout).  If you need to save the data files to a different location, see the Docker Advanced section.</li></ul><p>Other lines in this file you can ignore for now. </p><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>If you run into issues, try removing all the comments in the override file as they may conflict with the parsing done by docker-compose, though we have not found this to be a problem.</p></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="5-choose-pipeline-version"></a>5. Choose Pipeline Version<a class="hash-link" href="#5-choose-pipeline-version" title="Direct link to heading">#</a></h3><p>Once you&#x27;ve created the docker-compose.override.xml file and finished adjusting it for any customizations, you&#x27;re ready to select which image versions Docker should run.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>When prompted, select the <strong>same version</strong> you checked out earlier. </p><p>This script will replace the version numbers of docker images in docker-compose.override.yml and docker-compose.yml with the correct values.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="6-create-environment-file"></a>6. Create Environment File<a class="hash-link" href="#6-create-environment-file" title="Direct link to heading">#</a></h3><p></p><p>Next, copy <code>env.example</code> to <code>.env</code>  </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cp env.example .env </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>then edit the .env file to set the sensor names to unique identifiers (with spaces or not, no quotes)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># Importer settings</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflowSensorName=My sflow sensor name</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflowSensorName=My netflow sensor name</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ul><li>If you have only one collector, remove or comment out the line for the one you are not using.</li><li>If you have more than one of the same type of collector, see the &quot;Docker Advanced&quot; documentation.</li></ul><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>These names uniquely identify the source of the data and will be shown in the Grafana dashboards. In elasticsearch, they are saved in the <code>meta.sensor_id</code> field. Choose names that are meaningful and unique.
 For example, your sensor names might be &quot;MyNet New York Sflow&quot; and &quot;MyNet Boston Netflow&quot; or &quot;MyNet New York - London&quot; and &quot;MyNet New York - Paris&quot;. Whatever makes sense in your situation.</p></div></div><p>You will also want to edit the <strong>Logstash output rabbit queue</strong> section. This section defines where the final data will land after going through the pipeline.  By default, it will be written to a rabbitmq queue on <code>rabbit</code>, ie, the local rabbitMQ server running in the docker container. Enter a hostname to send to a remote rabbitMQ server (also the correct username, password, and queue key/name). </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_host=rabbit@mynet.edu</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_username=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_pw=guest</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">rabbitmq_output_key=netsage_archive_input</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-note alert alert--secondary"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="16" viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</h5></div><div class="admonition-content"><p>To send processed flow data to GlobalNOC at Indiana University, you will need to obtain settings for this section from your contact. A new queue may need to be set up at IU, as well as allowing traffic from your pipeline host. (At IU, data from the this final rabbit queue will be moved into an Elasticsearch instance for storage and viewing.)</p></div></div><p>The following options are described in the Docker Advanced section:</p><p><strong>To drop all flows except those using the specfied interfaces</strong>: Use if only some flows from a router are of interest and those can be identified by interface.</p><p><strong>To change the sensor name for flows using a certain interface</strong>: Use if you want to break out some flows coming into a port and give them a different sensor name.</p><p><strong>To &quot;manually&quot; correct flow sizes and rates for sampling for specified sensors</strong>: Use if sampling corrections are not being done automatically. Normally you do not need to use this, but check flows to be sure results are reasonable.</p><p></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="testing-the-collectors"></a>Testing the Collectors<a class="hash-link" href="#testing-the-collectors" title="Direct link to heading">#</a></h2><p>At this point, you can start the two flow collectors by themselves by running the following line. If you only need one of the collectors, remove the other from this command.  </p><p>(See the next section for how to start all the containers, including the collectors.)</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d sflow-collector netflow-collector</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Subdirectories for sflow/netflow, year, month, and day are created automatically under <code>data/input_data/</code>. File names contain dates and times.
 These are not text files; to view the contents, use an <a href="http://www.linuxcertif.com/man/1/nfdump/" target="_blank" rel="noopener noreferrer">nfdump command</a> (you will need to install nfdump).
 Files will be deleted automatically by the importer as they age out (the default is to keep 3 days).  </p><p>If the collector(s) are running properly, you should see nfcapd files being written every 5 minutes and they should have sizes of more than a few hundred bytes. (Empty files still have header and footer lines.)<br>
-See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/bare_metal_install"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Manual Installation Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/734dc415.80010e23.js"></script>
+See Troubleshooting if you have problems.</p><p>To stop the collectors</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="running-the-collectors-and-pipeline"></a>Running the Collectors and Pipeline<a class="hash-link" href="#running-the-collectors-and-pipeline" title="Direct link to heading">#</a></h2><p></p><p>Start up the pipeline (all containers) using:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will also restart any containers/processes that have died. &quot;-d&quot; runs containers in the background.</p><p>You can see the status of the containers and whether any have died (exited) using</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose ps</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check the logs for each of the containers, run</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs logstash</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose logs importer</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">etc.</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Add <code>-f</code> or, e.g., <code>-f logstash</code> to see new log messages as they arrive.  <code>--timestamps</code>, <code>--tail</code>,  and <code>--since</code> are also useful -- look up details in Docker documentation.</p><p>To shut down the pipeline (all containers) use</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"># docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Run all commands from the netsage-pipeline/ directory.</p><p></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_install_simple.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/elastic"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Elasticsearch</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Advanced Options Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-set-up-data-sources" class="table-of-contents__link">1. Set up Data Sources</a></li><li><a href="#2-set-up-a-pipeline-host" class="table-of-contents__link">2. Set up a Pipeline Host</a></li><li><a href="#3-clone-the-netsage-pipeline-project" class="table-of-contents__link">3. Clone the Netsage Pipeline Project</a></li><li><a href="#4-create-docker-composeoverrideyml" class="table-of-contents__link">4. Create Docker-compose.override.yml</a></li><li><a href="#5-choose-pipeline-version" class="table-of-contents__link">5. Choose Pipeline Version</a></li><li><a href="#6-create-environment-file" class="table-of-contents__link">6. Create Environment File</a></li><li><a href="#testing-the-collectors" class="table-of-contents__link">Testing the Collectors</a></li><li><a href="#running-the-collectors-and-pipeline" class="table-of-contents__link">Running the Collectors and Pipeline</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/734dc415.9fadde42.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/deploy/docker_troubleshoot/index.html b/docs/next/deploy/docker_troubleshoot/index.html
index e3c738ad..aad613a0 100644
--- a/docs/next/deploy/docker_troubleshoot/index.html
+++ b/docs/next/deploy/docker_troubleshoot/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Troubleshooting | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Docker Troubleshooting | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Troubleshooting"><meta data-react-helmet="true" property="og:description" content="Troubleshooting"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/653602f1.c1abe2fd.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/653602f1.d686a47d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
-likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/653602f1.c1abe2fd.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_troubleshoot">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Troubleshooting</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="troubleshooting"></a>Troubleshooting<a class="hash-link" href="#troubleshooting" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-you-are-not-seeing-flows-after-installation"></a>If you are not seeing flows after installation<a class="hash-link" href="#if-you-are-not-seeing-flows-after-installation" title="Direct link to heading">#</a></h3><p><strong>Troubleshooting checklist:</strong></p><ul><li>Use <code>docker-compose ps</code> to be sure the collectors (and other containers) are running.</li><li>Make sure you configured your routers to point to the correct address/port where the collector is running.  </li><li>Check iptables on your pipeline host to be sure incoming traffic from the routers is allowed.</li><li>Check to see if nfcapd files are being written. There should be a directory for the year, month, and day in netsage-pipeline/data/input_data/netflow/ or sflow/, and files should be larger than a few hundred bytes. If the files exist but are too small, the collector is running but there are no incoming flows.  &quot;nfdump -r filename&quot; will show the flows in a file (you may need to install nfdump).</li><li>Make sure you created .env and docker-compose.override.yml files and updated the settings accordingly,  sensorName especially since that identifies the source of the data.</li><li>Check the logs of the various containers to see if anything jumps out as being invalid.  <code>docker-compose logs $service</code>, where $service is logstash, importer, rabbit, etc.</li><li>If the final rabbit queue is on an external host, check the credentials you are using and whether iptables on that host allows incoming traffic from your pipeline host.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-flow-collection-stops"></a>If flow collection stops<a class="hash-link" href="#if-flow-collection-stops" title="Direct link to heading">#</a></h3><p><strong>Errors:</strong></p><ul><li>See if any of the containers has died using  <code>docker ps</code></li><li>Check the logs of the various containers to see if anything jumps out as being invalid. Eg, <code>docker-compose logs logstash</code>.</li><li>If logstash dies with an error about not finding *.conf files, make sure conf-logstash/ and directories and files within are readable by everyone (and directories are executable by everyone). The data/ directory and subdirectories need to be readable and writable by everyone, as well.</li></ul><p><strong>Disk space:</strong></p><ul><li>If the pipeline suddenly fails, check to see if the disk is full. If it is, first try getting rid of old docker images and containers to free up space: <code>docker image prune -a</code> and <code>docker container prune</code>.</li><li>Also check to see how much space the nfcapd files are consuming. You may need to add more disk space. You could also try automatically deleting nfcapd files after a fewer number of days (see Docker Advanced). </li></ul><p><strong>Memory:</strong></p><ul><li>If you are running a lot of data, sometimes docker may need to be allocated more memory. The most
+likely culprit is logstash (java) which is only allocated 2GB of RAM by default. Please see the Docker Advanced guide for how to change.</li></ul></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_troubleshooting.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_upgrade"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Upgrading</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/dev_dataset"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Replay Dataset »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#troubleshooting" class="table-of-contents__link">Troubleshooting</a><ul><li><a href="#if-you-are-not-seeing-flows-after-installation" class="table-of-contents__link">If you are not seeing flows after installation</a></li><li><a href="#if-flow-collection-stops" class="table-of-contents__link">If flow collection stops</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/653602f1.d686a47d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/deploy/docker_upgrade/index.html b/docs/next/deploy/docker_upgrade/index.html
index 3495b580..d124ecf6 100644
--- a/docs/next/deploy/docker_upgrade/index.html
+++ b/docs/next/deploy/docker_upgrade/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Upgrading | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Upgrading | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><meta data-react-helmet="true" property="og:description" content="To upgrade a previous installment of the Dockerized pipeline, perform the following steps."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/ceb45c47.a3330b5f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/ceb45c47.e6acb003.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull origin master</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these three commands to select the new release you want to run. In the first, replace &quot;{tag}&quot; by the version to run (eg, v1.2.11). When asked by the third, select the same version as the tag you checked out.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/ceb45c47.a3330b5f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/deploy/docker_upgrade">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/deploy/docker_upgrade">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Upgrading</h1></header><div class="markdown"><p>To upgrade a previous installment of the Dockerized pipeline, perform the following steps.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="shut-things-down"></a>Shut things down<a class="hash-link" href="#shut-things-down" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd {netsage-pipeline directory}</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose down</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will stop and remove all the docker containers, including the importer, logstash, and any collectors. Note that incoming flow data will not be saved during the time the collectors are down.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-source-code"></a>Update Source Code<a class="hash-link" href="#update-source-code" title="Direct link to heading">#</a></h3><p>To upgrade to a new release, pull new tags/code from github and docker images from dockerhub. Your customized .env and override files will not be overwritten, nor will data files, cache files, or downloaded support files. </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git reset --hard</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git fetch</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><div class="admonition admonition-warning alert alert--danger"><div class="admonition-heading"><h5><span class="admonition-icon"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16"><path fill-rule="evenodd" d="M5.05.31c.81 2.17.41 3.38-.52 4.31C3.55 5.67 1.98 6.45.9 7.98c-1.45 2.05-1.7 6.53 3.53 7.7-2.2-1.16-2.67-4.52-.3-6.61-.61 2.03.53 3.33 1.94 2.86 1.39-.47 2.3.53 2.27 1.67-.02.78-.31 1.44-1.13 1.81 3.42-.59 4.78-3.42 4.78-5.56 0-2.84-2.53-3.22-1.25-5.61-1.52.13-2.03 1.13-1.89 2.75.09 1.08-1.02 1.8-1.86 1.33-.67-.41-.66-1.19-.06-1.78C8.18 5.31 8.68 2.45 5.05.32L5.03.3l.02.01z"></path></svg></span>warning</h5></div><div class="admonition-content"><p>git reset --hard will obliterate any changes you have made to non-override files, eg, logstash conf files.  If necessary, please make sure you commit and save to a feature branch before continuing.</p></div></div><p>Run these commands to upgrade to the current announced version. While older versions of the pipeline are available, they are not recommended or supported for daily use. If in doubt, the Master branch contains the latest version. The documentation version at the top of this page is also the current pipeline version number.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git switch {tag} </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git pull</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./Upgrade_x_x_x.sh (where x_x_x is the current version you are upgrading to)</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>The docker-compose.yml and docker-compose.override.yml should both now have the version number you selected for pipeline_importer and pipeline_logstash.  </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="checkupdate-customization-files"></a>Check/Update Customization Files<a class="hash-link" href="#checkupdate-customization-files" title="Direct link to heading">#</a></h3><p>Occasionally, something may change which will necessitate editing your override and/or env file.</p><ul><li><p>Compare the new <code>docker-compose.override_example.yml</code> file to your <code>docker-compose.override.yml</code>. Be sure to check to see if the version of nfdump has changed. Look for lines like <code>image: netsage/nfdump-collector:</code>. Make sure the version in your override file matches what is the example file. (You do not need to actually perform any upgrade yourself. This will ensure the correct version is pulled from Docker Hub.) </p></li><li><p>Also, look for<code>version: &quot;x.x&quot;</code> at the top. If the version number is different, change it in your docker-compose.override.yml file. (This is the Compose file format version.)</p></li></ul><ul><li><p>Compare your <code>.env</code> file with the new <code>env.example</code> file to see if any new lines or sections have been added. If there have been any changes relevant to your deployment, eg, new options you want to use, copy the changes into your .env file. </p></li><li><p>If you used the Docker Advanced guide to make a <code>netsage_override.xml</code> file, compare it to <code>netsage_shared.xml</code> to see if there are any changes. This is unlikely.</p></li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="update-docker-containers"></a>Update Docker Containers<a class="hash-link" href="#update-docker-containers" title="Direct link to heading">#</a></h3><p>This should be done automatically when you start up the containers, but you can also pull new images from Docker Hub now.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose pull</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="restart-all-the-docker-containers"></a>Restart all the Docker Containers<a class="hash-link" href="#restart-all-the-docker-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>This will start all the services/containers listed in the docker-compose.yml and docker-compose.override.yml files, including the importer, logstash pipeline, and collectors.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="delete-old-images-and-containers"></a>Delete old images and containers<a class="hash-link" href="#delete-old-images-and-containers" title="Direct link to heading">#</a></h3><p>To keep things tidy, delete any old images and containers that are not being used.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image prune -a</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker container prune</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To check which images you have</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker image ls</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/deploy/docker_upgrade.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Advanced Options Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Troubleshooting »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#shut-things-down" class="table-of-contents__link">Shut things down</a></li><li><a href="#update-source-code" class="table-of-contents__link">Update Source Code</a></li><li><a href="#checkupdate-customization-files" class="table-of-contents__link">Check/Update Customization Files</a></li><li><a href="#update-docker-containers" class="table-of-contents__link">Update Docker Containers</a></li><li><a href="#restart-all-the-docker-containers" class="table-of-contents__link">Restart all the Docker Containers</a></li><li><a href="#delete-old-images-and-containers" class="table-of-contents__link">Delete old images and containers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/ceb45c47.e6acb003.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/devel/dev_dataset/index.html b/docs/next/devel/dev_dataset/index.html
index 27439744..c3181b13 100644
--- a/docs/next/devel/dev_dataset/index.html
+++ b/docs/next/devel/dev_dataset/index.html
@@ -6,31 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Pipeline Replay Dataset | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Pipeline Replay Dataset | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><meta data-react-helmet="true" property="og:description" content="The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/fa16f0fc.a192e853.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/fa16f0fc.8db267a7.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/dev_dataset">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/dev_dataset">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/dev_dataset">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/dev_dataset">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/dev_dataset">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/dev_dataset">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/dev_dataset">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/dev_dataset">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Pipeline Replay Dataset</h1></header><div class="markdown"><p>The Netsage Pipeline processes network data.  Though there are some components and patterns we can use to test
 the behavior using things like the Ruby unit <a href="https://github.com/netsage-project/netsage-pipeline/tree/master/conf-logstash/ruby/spec" target="_blank" rel="noopener noreferrer">tests</a> in logstash, and the <a href="https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html" target="_blank" rel="noopener noreferrer">generator</a> pligin, but the best
-test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/next/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/fa16f0fc.a192e853.js"></script>
+test is to replay network data and inspect the output in the grafana dashboard. </p><p>Two sample data set are provided for the two types of collectors we have (Netflow and Sflow).  The network data and ips have been anonymized and should have no identifying information. </p><p>You can download the files from <a href="https://drive.google.com/drive/folders/19fzY5EVoKwtYUaiBJq5OxAR82yDY0taG" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Please take note of which ports the collectors are listing on.  Check your docker-compose.override.yml file. If you are using default ports, they should match this <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/docker-compose.override_example.yml" target="_blank" rel="noopener noreferrer">example</a>. </p><p>Currently the default ports are:</p><ul><li>9998/udp for sflow </li><li>9999/udp for netflow</li></ul><p>Naturally the collectors have to be running in order for any of this to be usable.  You can read more on how to get them running in the <a href="/netsage-pipeline/docs/next/deploy/docker_install_simple#running-the-collectors">Docker Simple Deployment Guide</a>  </p><p>In order to replay the data, use the following commands for netflow and sflow respectively:</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netflow"></a>Netflow<a class="hash-link" href="#netflow" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">nfreplay -H 127.0.0.1 -p 9999  -r nfcapd-ilight-anon-20200114 -v 9 -d 1000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="sflow"></a>Sflow<a class="hash-link" href="#sflow" title="Direct link to heading">#</a></h3><p>Coming soon.  nfreplay will not work with sflow data type.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/pipeline_dataset.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Troubleshooting</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Dev Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netflow" class="table-of-contents__link">Netflow</a></li><li><a href="#sflow" class="table-of-contents__link">Sflow</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/fa16f0fc.8db267a7.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/devel/docker_dev_guide/index.html b/docs/next/devel/docker_dev_guide/index.html
index 5f543fc2..dc8f2989 100644
--- a/docs/next/devel/docker_dev_guide/index.html
+++ b/docs/next/devel/docker_dev_guide/index.html
@@ -6,32 +6,31 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Docker Dev Guide | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Docker Dev Guide | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Selecting a Version"><meta data-react-helmet="true" property="og:description" content="Selecting a Version"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1f3a3353.c8d7370b.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1f3a3353.117952bc.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_guide">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_guide">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Docker Dev Guide</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="selecting-a-version"></a>Selecting a Version<a class="hash-link" href="#selecting-a-version" title="Direct link to heading">#</a></h2><p>You can use the &quot;master&quot; version or a tagged version.<br>
 To select a released version use the docker_select_version.sh script (see the Deployment Guide).
-If you wish to use the development version (master branch) simply skip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-images"></a>Build Images<a class="hash-link" href="#build-images" title="Direct link to heading">#</a></h2><p>The images are published on Docker Hub, but if you&#x27;d like to incorporate local changes please follow the process below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="build-using-source-code"></a>Build Using Source Code<a class="hash-link" href="#build-using-source-code" title="Direct link to heading">#</a></h3><p>If you would like to build the <em>importer</em> container using the version of the pipeline scripts found in the GitHub repo then run the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain" style="display:inline-block">
-</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>NOTE: The importer container includes the config files for the logstash pipeline. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="optional-elasticsearch-and-kibana"></a>Optional: ElasticSearch and Kibana<a class="hash-link" href="#optional-elasticsearch-and-kibana" title="Direct link to heading">#</a></h2><p>You can optionally store flow data locally in an ElasticSearch container and view the data with Kibana. Local storage can be enabled with the following steps:</p><ol><li>Uncomment the following lines in conf-logstash/99-outputs.conf:</li></ol><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">elasticsearch {</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    hosts =&gt; [&quot;elasticsearch&quot;]</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    index =&gt; &quot;netsage_flow-%{+YYYY.MM.dd}&quot;</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">}</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><ol start="2"><li><p>Comment out the <code>rabbitmq {...}</code> block in conf-logstash/99-outputs.conf if you do not want to also send logstash output to RabbitMQ.</p></li><li><p>Run the containers using the following line: <code> </code>  <code>docker-compose -f docker-compose.yml  -f docker-compose.develop.yml up  -d</code>  <code> </code></p></li></ol><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#build-images" class="table-of-contents__link">Build Images</a><ul><li><a href="#build-using-source-code" class="table-of-contents__link">Build Using Source Code</a></li></ul></li><li><a href="#optional-elasticsearch-and-kibana" class="table-of-contents__link">Optional: ElasticSearch and Kibana</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/1f3a3353.c8d7370b.js"></script>
+If you wish to use the development version (master branch) simply skip the docker_select_version.sh step.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installing"></a>Installing<a class="hash-link" href="#installing" title="Direct link to heading">#</a></h2><p>See the Deployment Guide to learn how to set up collectors, your environment and override files, etc.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="importer"></a>Importer<a class="hash-link" href="#importer" title="Direct link to heading">#</a></h2><p>The importer &quot;shared&quot; config that Docker uses is defined in compose/netsage_shared.xml.  <strong> NOTE: If you want to make changes to this file, you will need to rebuild the container</strong></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="handy-docker-commands"></a>Handy Docker Commands<a class="hash-link" href="#handy-docker-commands" title="Direct link to heading">#</a></h2><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="start-the-containers"></a>Start the Containers<a class="hash-link" href="#start-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose up -d </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="stop-the-containers"></a>Stop the Containers<a class="hash-link" href="#stop-the-containers" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose stop &amp;&amp; docker-compose rm </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="enter-a-container-shell"></a>Enter a Container Shell<a class="hash-link" href="#enter-a-container-shell" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec logstash bash     #bash shell in logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec importer bash     #bash shell in importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose exec rabbit bash       #bash shell in rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="view-container-logs"></a>View Container Logs<a class="hash-link" href="#view-container-logs" title="Direct link to heading">#</a></h3><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly sh"><div tabindex="0" class="prism-code language-sh codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f              #view logs for all containers </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f logstash     #view logs for logstash container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f importer     #view logs for importer container</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">docker-compose logs -f rabbit       #view logs for rabbit container</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/docker.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/dev_dataset"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Pipeline Replay Dataset</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docusaurus"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Revising Documentation »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#selecting-a-version" class="table-of-contents__link">Selecting a Version</a></li><li><a href="#installing" class="table-of-contents__link">Installing</a></li><li><a href="#importer" class="table-of-contents__link">Importer</a></li><li><a href="#handy-docker-commands" class="table-of-contents__link">Handy Docker Commands</a><ul><li><a href="#start-the-containers" class="table-of-contents__link">Start the Containers</a></li><li><a href="#stop-the-containers" class="table-of-contents__link">Stop the Containers</a></li><li><a href="#enter-a-container-shell" class="table-of-contents__link">Enter a Container Shell</a></li><li><a href="#view-container-logs" class="table-of-contents__link">View Container Logs</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/1f3a3353.117952bc.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/devel/docker_dev_tag/index.html b/docs/next/devel/docker_dev_tag/index.html
index 20149cf9..57d38b85 100644
--- a/docs/next/devel/docker_dev_tag/index.html
+++ b/docs/next/devel/docker_dev_tag/index.html
@@ -6,32 +6,32 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">How to Release a New Version of the Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="How to Release a New Version of the Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><meta data-react-helmet="true" property="og:description" content="If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipelineimporter and pipelinelogstash images.."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/16fcefd4.2fe48b23.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/16fcefd4.9eb05a6f.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docker_dev_tag">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docker_dev_tag">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">How to Release a New Version of the Pipeline</h1></header><div class="markdown"><p>If a new version of nfdump needs to be used, make the new nfdump-collector image(s) first (see below) and update the docker-compose files with the new version number, then make new pipeline_importer and pipeline_logstash images..</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-an-rpm-release"></a>Make an RPM Release<a class="hash-link" href="#make-an-rpm-release" title="Direct link to heading">#</a></h2><p>Use standard procedures to create an rpm of the new version of the pipeline. Update the version number and the CHANGES file, build the rpm, repoify, etc., then upgrade grnoc-netsage-deidentifier on bare-metal hosts using yum. If all works well, do the following steps to create new Docker images with which to upgrade Docker deployments.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="in-github-create-a-release-tag"></a>In Github, Create a Release Tag<a class="hash-link" href="#in-github-create-a-release-tag" title="Direct link to heading">#</a></h2><p>Create a new Tag or Release in Github, eg, v1.2.11.
 Be sure to copy info from the CHANGES file into the Release description.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-build-and-push-images-manually"></a>To Build and Push Images Manually<a class="hash-link" href="#to-build-and-push-images-manually" title="Direct link to heading">#</a></h2><p>Below is the procedure to build pipeline_importer and pipeline_logstash images manually.</p><p>Install docker-compose if not done already. See the Docker Installation instructions.</p><p>Git clone (or git pull) the pipeline project and check out the tag you want to build, then set the version number in docker-compose.build.yml using the script. Eg, for v1.2.11,</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">git clone https://github.com/netsage-project/netsage-pipeline.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">cd netsage-pipeline</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">git checkout -b v1.2.11</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">./scripts/docker_select_version.sh 1.2.11</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Then build the pipeline_importer and pipeline_logstash images and push them to Docker Hub:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml build</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker-compose -f docker-compose.build.yml push    (will push images mentioned in docker-compose.yml ??)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     or  $ docker push $image:$tag                        (will push a specific image version)</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>If you run into an error about retrieving a mirrorlist and could not find a valid baseurl for repo, restart docker and try again.
 If that doesn&#x27;t work, try adding this to /etc/hosts: <code>67.219.148.138  mirrorlist.centos.org</code>, and/or try <code>yum install net-tools bridge-utils</code>, and/or restart network.service then docker. </p><p>The person pushing to Docker Hub must have a Docker Hub account and belong to the Netsage team (3 users are allowed, for the free level).</p><p>It might be a good idea to test the images before pushing them. See &quot;Test Docker Images&quot; below.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="building-with-automation"></a>Building With Automation<a class="hash-link" href="#building-with-automation" title="Direct link to heading">#</a></h2><p>???</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="test-docker-images"></a>Test Docker Images<a class="hash-link" href="#test-docker-images" title="Direct link to heading">#</a></h2><p>See the Docker installation instructions for details... </p><p>In the git checkout of the correct version, make an .env file and a docker-compose.override.yml file. You probably want to send the processed data to a dev Elasticsearch instance. Use samplicate or some other method to have data sent to the dev host. </p><p>Run docker_select_version.sh if you haven&#x27;t already, then start it up <code>$ sudo docker-compose up -d</code>. If there are local images, they&#x27;ll be used, otherwise they&#x27;ll be pulled from Docker Hub.</p><p>After about 30 minutes, you should see flows in elasticsearch.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="make-versioned-docs"></a>Make Versioned Docs<a class="hash-link" href="#make-versioned-docs" title="Direct link to heading">#</a></h2><p>A new set of versioned docs also has to be tagged once you are done making changes for the latest pipeline version. See the <strong>Docusaurus guide</strong>. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-new-nfdump-collector-images"></a>To Make New Nfdump-Collector Images<a class="hash-link" href="#to-make-new-nfdump-collector-images" title="Direct link to heading">#</a></h2><p>If a new version of nfdump has been released that we need, new nfdump-collector images need to be made.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ git clone https://github.com/netsage-project/docker-nfdump-collector.git</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd docker-nfdump-collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl start docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use squash: create a file at /etc/docker/daemon.json and put into it</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;experimental&quot;: true  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain"> &quot;debug: false&quot;</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To build version $VER, eg, 1.6.23 (both regular and alpine linux versions ?):</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:$VER --squash  collector</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker build --build-arg NFDUMP_VERSION=$VER  --tag netsage/nfdump-collector:alpine-$VER -f collector/Dockerfile-alpine --squash .    </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To push to Docker Hub and quit docker</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker login</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">     provide your DockerHub login credentials</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo docker push netsage/nfdump-collector:$VER</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo systemctl stop docker</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>To use the new collector image in the pipeline, change the version number in docker-compose.override_example.yml. For example, to use the alpine-1.6.23 image:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">sflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">...</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">netflow-collector:</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">    image: netsage/nfdump-collector:alpine-1.6.23</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Remind users to make the same change in their docker-compose.override.yml file when they do the next pipeline upgrade.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="new-version-of-logstash"></a>New Version of Logstash<a class="hash-link" href="#new-version-of-logstash" title="Direct link to heading">#</a></h3><p>If a new version of logstash has been released that we want everyone to use,
-???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/16fcefd4.2fe48b23.js"></script>
+???</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/tag.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docusaurus"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Revising Documentation</div></a></div><div class="pagination-nav__item pagination-nav__item--next"></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#make-an-rpm-release" class="table-of-contents__link">Make an RPM Release</a></li><li><a href="#in-github-create-a-release-tag" class="table-of-contents__link">In Github, Create a Release Tag</a></li><li><a href="#to-build-and-push-images-manually" class="table-of-contents__link">To Build and Push Images Manually</a></li><li><a href="#building-with-automation" class="table-of-contents__link">Building With Automation</a></li><li><a href="#test-docker-images" class="table-of-contents__link">Test Docker Images</a></li><li><a href="#make-versioned-docs" class="table-of-contents__link">Make Versioned Docs</a></li><li><a href="#to-make-new-nfdump-collector-images" class="table-of-contents__link">To Make New Nfdump-Collector Images</a><ul><li><a href="#new-version-of-logstash" class="table-of-contents__link">New Version of Logstash</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/16fcefd4.9eb05a6f.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/devel/docusaurus/index.html b/docs/next/devel/docusaurus/index.html
index ae2edb2a..69cdfbf1 100644
--- a/docs/next/devel/docusaurus/index.html
+++ b/docs/next/devel/docusaurus/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Revising Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Revising Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="This project&#x27;s documentation uses Docusaurus."><meta data-react-helmet="true" property="og:description" content="This project&#x27;s documentation uses Docusaurus."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/4ffdf56a.031ac316.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/4ffdf56a.f556f662.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/devel/docusaurus">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/devel/docusaurus">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/devel/docusaurus">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/devel/docusaurus">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/devel/docusaurus">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Revising Documentation</h1></header><div class="markdown"><p>This project&#x27;s documentation uses Docusaurus.</p><p>Docusaurus converts markdown into html and builds a static website using React UI components, which can be exported to a webserver.</p><p>Yarn is a package manager for JavaScript and replaces the npm client. It is not strictly necessary but highly encouraged.</p><p>To extend the docs simply create a markdown file and reference the ID in the side bar config. Please see the related documentation
 at the <a href="https://v2.docusaurus.io/" target="_blank" rel="noopener noreferrer">docusaurus 2</a> project website.</p><p><em>THE FOLLOWING INSTRUCTIONS ARE NOT CONFIRMED TO WORK. PLEASE UPDATE WITH CORRECTIONS.</em></p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-not-using-docker"></a>If Not Using Docker<a class="hash-link" href="#if-not-using-docker" title="Direct link to heading">#</a></h2><p>These are instructions for editing and releasing docs without using Docker.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="installation"></a>Installation<a class="hash-link" href="#installation" title="Direct link to heading">#</a></h3><p>To get started the first time, install npm, then use that to install yarn </p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo yum install npm</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ sudo npm install -g yarn </span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>Git clone the netsage pipeline project, then run yarn install to get all the dependencies listed within package.json</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ cd netsage-pipeline/website</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn install</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="if-local-development"></a>If Local Development<a class="hash-link" href="#if-local-development" title="Direct link to heading">#</a></h3><p>If you are working on your local machine, rather than sshing into a host, you can view changes to the docs in a browser as you work. Use the following commands to generate the static website content (gets written into the build directory), then start a local development server and open up a browser window in which to view the docs. Most changes you make will be reflected live without having to restart the server.</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn build  </span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn start</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">go to http://localhost:3000</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="to-make-changes"></a>To Make Changes<a class="hash-link" href="#to-make-changes" title="Direct link to heading">#</a></h3><p>Whether on a local machine or a linux host, to make changes, edit the files in website/docs/.
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><p>To view the changes you&#x27;ve made with some formatting, just go to the file on github in a browser. To see all of the formatting, read the &quot;Deploying Docs to github.io&quot; section below.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ yarn run docusaurus docs:version a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
@@ -28,16 +28,16 @@
 When finished, git add, git commit, git push, as usual.
 Repeat as needed.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tagging-a-new-release-1"></a>Tagging a New release<a class="hash-link" href="#tagging-a-new-release-1" title="Direct link to heading">#</a></h3><p>When it&#x27;s time to release a new version of the Pipeline, you need to create a new version of the docs as well. </p><p>Once the documentation is stable and you don&#x27;t forsee any new change, please do the following:</p><div class="codeBlockContainer_K1bP"><div class="codeBlockContent_hGly"><div tabindex="0" class="prism-code language-undefined codeBlock_23N8 thin-scrollbar"><div class="codeBlockLines_39YC" style="color:#bfc7d5;background-color:#292d3e"><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose build build_docs</span></div><div class="token-line" style="color:#bfc7d5"><span class="token plain">$ docker-compose run  docs yarn run docusaurus docs:version  a.b.c</span></div></div></div><button type="button" aria-label="Copy code to clipboard" class="copyButton_Ue-o">Copy</button></div></div><p>replacing a.b.c with the next release version number.<br>
 This will create new versioned docs in website/versioned_docs/.</p><p>Then edit docusaurus.config.js and change <code>lastVersion:</code> to refer to the new version number. </p><p>Finally, commit and push the following to github:</p><ul><li>website/versioned_docs/version-a.b.c/</li><li>website/versioned_sidebars/version-a.b.c.sidebars.json</li><li>versions.json </li><li>docusaurus.config.js</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="deploying-docs-to-githubio-1"></a>Deploying Docs to github.io<a class="hash-link" href="#deploying-docs-to-githubio-1" title="Direct link to heading">#</a></h3><p>How to do this when using Docker ??? Get into the container ???</p><p>For now, go a linux server that has yarn installed and
-follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/4ffdf56a.031ac316.js"></script>
+follow the instructions under If Not Using Docker.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/devel/documentation_guide.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_guide"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Docker Dev Guide</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/devel/docker_dev_tag"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">How to Release a New Version of the Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#if-not-using-docker" class="table-of-contents__link">If Not Using Docker</a><ul><li><a href="#installation" class="table-of-contents__link">Installation</a></li><li><a href="#if-local-development" class="table-of-contents__link">If Local Development</a></li><li><a href="#to-make-changes" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio" class="table-of-contents__link">Deploying Docs to github.io</a></li><li><a href="#removing-a-version" class="table-of-contents__link">Removing a version</a></li></ul></li><li><a href="#if-using-docker" class="table-of-contents__link">If Using Docker</a><ul><li><a href="#build-and-start-the-container" class="table-of-contents__link">Build and Start the Container</a></li><li><a href="#to-make-changes-1" class="table-of-contents__link">To Make Changes</a></li><li><a href="#tagging-a-new-release-1" class="table-of-contents__link">Tagging a New release</a></li><li><a href="#deploying-docs-to-githubio-1" class="table-of-contents__link">Deploying Docs to github.io</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/4ffdf56a.f556f662.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/elastic/index.html b/docs/next/pipeline/elastic/index.html
index 91c5989d..4bc313f4 100644
--- a/docs/next/pipeline/elastic/index.html
+++ b/docs/next/pipeline/elastic/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/7d0b7755.4b75443f.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/7d0b7755.c19774b1.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/7d0b7755.4b75443f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/elastic">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/7d0b7755.c19774b1.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/importer/index.html b/docs/next/pipeline/importer/index.html
index 7941e8cc..ec4f1b2c 100644
--- a/docs/next/pipeline/importer/index.html
+++ b/docs/next/pipeline/importer/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d3556f45.a168af5e.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/d3556f45.2aeec99a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d3556f45.a168af5e.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/importer">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/d3556f45.2aeec99a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/intro/index.html b/docs/next/pipeline/intro/index.html
index a5717899..80d26285 100644
--- a/docs/next/pipeline/intro/index.html
+++ b/docs/next/pipeline/intro/index.html
@@ -6,30 +6,30 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/22b7278f.fe1cf5a5.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/22b7278f.58242775.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/next/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/next/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/22b7278f.fe1cf5a5.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/intro">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/next/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/next/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/22b7278f.58242775.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/logstash/index.html b/docs/next/pipeline/logstash/index.html
index 13d79dd9..3878656b 100644
--- a/docs/next/pipeline/logstash/index.html
+++ b/docs/next/pipeline/logstash/index.html
@@ -6,20 +6,20 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/bd6c6f88.e87a9bd6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/bd6c6f88.8d988769.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/logstash">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 sets duration and rates to 0 if duration is &lt;= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="15-sensor-specific-changesconf"></a>15-sensor-specific-changes.conf<a class="hash-link" href="#15-sensor-specific-changesconf" title="Direct link to heading">#</a></h3><p>Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="20-add_idconf"></a>20-add_id.conf<a class="hash-link" href="#20-add_idconf" title="Direct link to heading">#</a></h3><p>Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="40-aggregationconf"></a>40-aggregation.conf<a class="hash-link" href="#40-aggregationconf" title="Direct link to heading">#</a></h3><p>Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow.</p><p>Notes: </p><ul><li>By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together.</li><li>If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes.</li><li>There is another &quot;timeout&quot; setting which is basically the maximum duration of a stitched flow (default: 24 hr).</li><li>When logstash shuts down, any flows &quot;in the aggregator&quot; will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. </li><li>Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file.</li><li>Tstat flows come in already complete, so no aggregation is done on those flows.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="45-geoip-taggingconf"></a>45-geoip-tagging.conf<a class="hash-link" href="#45-geoip-taggingconf" title="Direct link to heading">#</a></h3><p>Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;
@@ -27,16 +27,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/bd6c6f88.e87a9bd6.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/bd6c6f88.8d988769.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/nfdump/index.html b/docs/next/pipeline/nfdump/index.html
index 0bbcb577..e08a6655 100644
--- a/docs/next/pipeline/nfdump/index.html
+++ b/docs/next/pipeline/nfdump/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Sflow/Netflow Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Sflow/Netflow Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1cc43674.8da06d41.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1cc43674.f9080c78.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/next/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/next/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/1cc43674.8da06d41.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/nfdump">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/next/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/next/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/1cc43674.f9080c78.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/pipeline/tstat/index.html b/docs/next/pipeline/tstat/index.html
index 76afe344..3e6d17a2 100644
--- a/docs/next/pipeline/tstat/index.html
+++ b/docs/next/pipeline/tstat/index.html
@@ -6,29 +6,29 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="current"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-current"><meta data-react-helmet="true" property="og:title" content="Tstat Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage GitHub Project"><meta data-react-helmet="true" property="og:description" content="Netsage GitHub Project"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/e2c161d9.57d50314.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/935f2afb.9f248d32.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/e2c161d9.00dc4bde.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.0.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/next/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/935f2afb.5c9a98e4.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/e2c161d9.57d50314.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/next/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/next/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for Netsage Pipeline Documentation <strong>master (unreleased)</strong> version.</div><div class="margin-top--md">For up-to-date documentation, see the <strong><a href="/netsage-pipeline/docs/pipeline/tstat">latest version</a></strong> (2.1.0).</div></div><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: master (unreleased)</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/next/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/docs/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/next/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/935f2afb.9f248d32.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/e2c161d9.00dc4bde.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/elastic/index.html b/docs/pipeline/elastic/index.html
index d8ff9f04..1bf468fb 100644
--- a/docs/pipeline/elastic/index.html
+++ b/docs/pipeline/elastic/index.html
@@ -5,30 +5,30 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9001d127.1e177d17.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Elasticsearch | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Elasticsearch | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><meta data-react-helmet="true" property="og:description" content="Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/71e5fd50.9e08f691.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/elastic">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/choose_install"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Choosing an Installation Procedure »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9001d127.1e177d17.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/elastic">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/elastic">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/elastic">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/elastic">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/elastic">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/elastic">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/elastic">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Elasticsearch</h1></header><div class="markdown"><p>Flow data is ultimately saved to Elasticsearch. Following are the fields that are used/created in Logstash and that you may see returned by an elasticsearch query.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="flow-fields"></a>Flow fields<a class="hash-link" href="#flow-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>start</td><td>Jun 9, 2020 @ 17:39:53.808</td><td>Start time of the flow (first packet seen)</td></tr><tr><td>end</td><td>Jun 9, 2020 @ 17:39:57.699</td><td>End time of the flow   (last packet seen)</td></tr><tr><td>meta.id</td><td>a17c4f0542...</td><td>Id of the flow (hash of 5-tuple + Sensor name)</td></tr><tr><td>es_doc_id</td><td>4f46bef884...</td><td>Hash of meta.id and start time. May be used as doc id in ES to prevent duplicates, but see Notes elsewhere.</td></tr><tr><td>meta.flow_type</td><td>sflow</td><td>&#x27;sflow&#x27;, &#x27;netflow&#x27;, or &#x27;tstat&#x27;</td></tr><tr><td>meta.protocol</td><td>tcp</td><td>Protocol used</td></tr><tr><td>meta.sensor_id</td><td>snvl2-pw-sw-1-mgmt-2.cenic.net</td><td>Sensor name (set in importer config, may not always be a hostname)</td></tr><tr><td>meta.sensor_group</td><td>CENIC</td><td>Sensor group, usually the network</td></tr><tr><td>meta.sensor_type</td><td>Regional Network</td><td>Sensor type (&#x27;Circuit&#x27;, &#x27;Regional Network&#x27;, etc)</td></tr><tr><td>meta.country_scope</td><td>Domestic</td><td>&#x27;Domestic&#x27;, &#x27;International&#x27;, or &#x27;Mixed&#x27;, depending on countries of src and dst</td></tr><tr><td>meta.is_network_testing</td><td>no</td><td>&#x27;yes&#x27; if discipline is &#x27;CS.Network Testing and Monitoring&#x27; or port is one used for PerfSonar: 5001, 5101, or 5201</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-fields-destination-fields-similarly-with-dst"></a>Source Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_ip</td><td>171.64.68.x</td><td>deidentified IP address</td></tr><tr><td>meta.src_port</td><td>80</td><td>port used</td></tr><tr><td>meta.src_asn</td><td>32</td><td>Source ASN from the flow header or, in some cases, the ANS of the IP from the MaxMind GeoIP ASN database</td></tr><tr><td>meta.src_organization</td><td>Stanford University</td><td>organization that owns the AS from the CAIDA ASN-Organization database</td></tr><tr><td>meta.src_location.lat</td><td>37.423</td><td>latitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_location.lon</td><td>-122.164</td><td>longitude of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_country_name</td><td>United States</td><td>country of the IP from the MaxMind GeoIP City database</td></tr><tr><td>meta.src_continent</td><td>North America</td><td>continent of the IP the MaxMind GeoIP City database</td></tr><tr><td>meta.src_ifindex</td><td>166</td><td>the index of the interface the flow came into</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-science-registry-fields--destination-fields-similarly-with-dst"></a>Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-science-registry-fields--destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><p>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</p><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.scireg.src.discipline</td><td>MPS.Physics.High Energy</td><td>The science discipline that uses the resource (ie IP). Note that  not the src MAY not have the same discipline as the dst.</td></tr><tr><td>meta.scireg.src.role</td><td>Storage</td><td>Role that the host plays</td></tr><tr><td>meta.scireg.src.org_name</td><td>Boston University (BU)</td><td>The organization the manages and/or uses the resource, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.org_abbr</td><td>Boston U</td><td>A shorter name for the organization. May not be the official abbreviation.</td></tr><tr><td>meta.scireg.src.resource</td><td>BU - ATLAS</td><td>Descriptive resource name from SciReg</td></tr><tr><td>meta.scireg.src.resource_abbr</td><td></td><td>Resource abbreviation (if any)</td></tr><tr><td>meta.scireg.src.project_names</td><td>ATLAS</td><td>&quot;Projects&quot; that the resource is part of</td></tr><tr><td>meta.scireg.src.latitude</td><td>37.4178</td><td>Resource&#x27;s latitude, as listed in the Science Registry</td></tr><tr><td>meta.scireg.src.longitude</td><td>-122.178</td><td>Resource&#x27;s longitude, as listed in the Science Registry</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="source-preferred-fields-destination-fields-similarly-with-dst"></a>Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)<a class="hash-link" href="#source-preferred-fields-destination-fields-similarly-with-dst" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>meta.src_preferred_org</td><td>Stanford University</td><td>If the IP was found in the Science Registry, this is the SciReg organization, otherwise it is the CAIDA organization</td></tr><tr><td>meta.src_preferred_location.lat</td><td>37.417800</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr><tr><td>meta.src_preferred_location.lon</td><td>-122.172000i</td><td>Science Registry value if available, otherwise the MaxMind City DB value</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="value-fields"></a>Value Fields<a class="hash-link" href="#value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>values.num_bits</td><td>939, 458, 560</td><td>Sum of the number of bits in the (stitched) flow</td></tr><tr><td>values.num_packets</td><td>77, 824</td><td>Sum of the number of packets in the (stitched) flows</td></tr><tr><td>values.duration</td><td>3.891</td><td>Calculated as end minus start.</td></tr><tr><td>values.bits_per_second</td><td>241, 443, 988</td><td>Calculated as num_bits divided by duration</td></tr><tr><td>values.packets_per_second</td><td>20, 001</td><td>Calculated as num_packets divided by duration</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="tstat-value-fields"></a>Tstat Value Fields<a class="hash-link" href="#tstat-value-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th></tr></thead><tbody><tr><td>values.tcp_cwin_max</td><td>1549681</td></tr><tr><td>values.tcp_cwin_min</td><td>17</td></tr><tr><td>values.tcp_initial_cwin</td><td>313</td></tr><tr><td>values.tcp_max_seg_size</td><td>64313</td></tr><tr><td>values.tcp_min_seg_size</td><td>17</td></tr><tr><td>values.tcp_mss</td><td>8960</td></tr><tr><td>values.tcp_out_seq_pkts</td><td>0</td></tr><tr><td>values.tcp_pkts_dup</td><td>0</td></tr><tr><td>values.tcp_pkts_fc</td><td>0</td></tr><tr><td>values.tcp_pkts_fs</td><td>0</td></tr><tr><td>values.tcp_pkts_reor</td><td>0</td></tr><tr><td>values.tcp_pkts_rto</td><td>0</td></tr><tr><td>values.tcp_pkts_unfs</td><td>0</td></tr><tr><td>values.tcp_pkts_unk</td><td>2</td></tr><tr><td>values.tcp_pkts_unrto</td><td>0</td></tr><tr><td>values.tcp_rexmit_bytes</td><td>1678</td></tr><tr><td>values.tcp_rexmit_pkts</td><td>2</td></tr><tr><td>values.tcp_rtt_avg</td><td>0.044</td></tr><tr><td>values.tcp_rtt_max</td><td>39.527</td></tr><tr><td>values.tcp_rtt_min</td><td>0.001</td></tr><tr><td>values.tcp_rtt_std</td><td>0.276</td></tr><tr><td>values.tcp_sack_cnt</td><td>1</td></tr><tr><td>values.tcp_win_max</td><td>1549681</td></tr><tr><td>values.tcp_win_min</td><td>17</td></tr><tr><td>values.tcp_window_scale</td><td>13</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="developer-fields"></a>Developer Fields<a class="hash-link" href="#developer-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>@pipeline_ver</td><td>1.2.11</td><td>Version number of the pipeline used to process this flow</td></tr><tr><td>@ingest_time</td><td>Jun 9, 2020 @ 10:03:20.700</td><td>The time the flow entered the logstash pipeline</td></tr><tr><td>@timestamp</td><td>Jun 9, 2020 @ 18:03:21.703</td><td>The time the flow entered the logstash pipeline for tstat flows, or the time stitching finished and the event exited the aggregation filter for other flows.</td></tr><tr><td>@exit_time</td><td>Jun 9, 2020 @ 18:03:25.369</td><td>The time the flow exited the pipeline</td></tr><tr><td>@processing_time</td><td>688.31</td><td>@exit_time minus @ingest_time. Useful for seeing how long stitching took.</td></tr><tr><td>stitched_flows</td><td>13</td><td>Number of flows that came into logstash that were stitched together to make this final one. 1 if no flows were stitched together. 0 for tstat flows, which are never stitched.</td></tr><tr><td>tags</td><td>maxmind src asn</td><td>Various info and error messages</td></tr><tr><td>trial</td><td>5</td><td>Can be set in 40-aggregation.conf if desired</td></tr></tbody></table><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="elasticsearch-fields"></a>Elasticsearch Fields<a class="hash-link" href="#elasticsearch-fields" title="Direct link to heading">#</a></h3><table><thead><tr><th>name</th><th>example</th><th>description</th></tr></thead><tbody><tr><td>_index</td><td>om-ns-netsage-2020.06.14</td><td>name of the index (&quot;database table&quot;)</td></tr><tr><td>_type</td><td>_doc</td><td>set by ES</td></tr><tr><td>_id</td><td>HRkcm3IByJ9fEnbnCpaY</td><td>elasticsearch document id.</td></tr><tr><td>_score</td><td>1</td><td>set by ES query</td></tr><tr><td>@version</td><td>1</td><td>set by ES</td></tr></tbody></table></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/elastic_search.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/logstash"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Logstash Pipeline</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/deploy/docker_install_simple"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Installation Guide »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#flow-fields" class="table-of-contents__link">Flow fields</a></li><li><a href="#source-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-science-registry-fields--destination-fields-similarly-with-dst" class="table-of-contents__link">Source Science Registry Fields  (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#source-preferred-fields-destination-fields-similarly-with-dst" class="table-of-contents__link">Source &quot;Preferred&quot; Fields (Destination Fields similarly with &quot;dst&quot;)</a></li><li><a href="#value-fields" class="table-of-contents__link">Value Fields</a></li><li><a href="#tstat-value-fields" class="table-of-contents__link">Tstat Value Fields</a></li><li><a href="#developer-fields" class="table-of-contents__link">Developer Fields</a></li><li><a href="#elasticsearch-fields" class="table-of-contents__link">Elasticsearch Fields</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/71e5fd50.9e08f691.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/importer/index.html b/docs/pipeline/importer/index.html
index 327b53b9..c844bbeb 100644
--- a/docs/pipeline/importer/index.html
+++ b/docs/pipeline/importer/index.html
@@ -5,31 +5,31 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/d1b2eded.fdea14ce.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Importer | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Importer | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><meta data-react-helmet="true" property="og:description" content="A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsagedeidentifierraw&quot; RabbitMQ queue."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/5ef24f1d.6c97f268.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/importer">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
-All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/d1b2eded.fdea14ce.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/importer">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/importer">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/importer">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/importer">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/importer">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/importer">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/importer">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Importer</h1></header><div class="markdown"><p>A netsage-netflow-importer script reads any new nfcapd files that have come in after a configurable delay and writes the results to the &quot;netsage_deidentifier_raw&quot; RabbitMQ queue.
+All flow data waits in the queue until it is read in and processed by the logstash pipeline.</p><p>To read nfcapd files, the importer uses an nfdump command with the &quot;-a&quot; option to aggregate raw flows within the file by the &quot;5-tuple,&quot; i.e., the source and destination IPs, ports, and protocol. The  &quot;-L&quot; option is used to throw out any aggregated flows below a threshold number of bytes. This threshold is specified in the importer config file. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="configuration"></a>Configuration<a class="hash-link" href="#configuration" title="Direct link to heading">#</a></h3><p>Configuration files for the importer are netsage_netflow_importer.xml and netsage_shared.xml in /etc/grnoc/netsage/deidentfier/. Comments in the files briefly describe the options. See also the Deployment pages in these docs.</p><p>To avoid re-reading nfcapd files, the importer stores the names of files that have already been read in /var/cache/netsage/netflow_importer.cache. </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/importer.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/nfdump"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Sflow/Netflow Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/logstash"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Logstash Pipeline »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#configuration" class="table-of-contents__link">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/5ef24f1d.6c97f268.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/intro/index.html b/docs/pipeline/intro/index.html
index ef66d74c..730f79b8 100644
--- a/docs/pipeline/intro/index.html
+++ b/docs/pipeline/intro/index.html
@@ -5,31 +5,31 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/af9cfdf8.a481c25f.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Intro | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Intro | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The NetSage Pipeline"><meta data-react-helmet="true" property="og:description" content="The NetSage Pipeline"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f2cce206.f146a973.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
-There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/af9cfdf8.a481c25f.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Intro</h1></header><div class="markdown"><h1><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="the-netsage-pipeline"></a>The NetSage Pipeline<a class="hash-link" href="#the-netsage-pipeline" title="Direct link to heading">#</a></h1><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="description"></a>Description<a class="hash-link" href="#description" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is composed of several components for processing network flow data, including importing, deidentification, metadata tagging, flow stitching, etc.
+There are many ways the components can be combined, configured, and run. These documents will describe the standard &quot;simple&quot; set up and provide information for more complex configurations.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="data-collection"></a>Data Collection<a class="hash-link" href="#data-collection" title="Direct link to heading">#</a></h2><p>In Netsage, sensor(s) are network devices configured to collect flow data (<a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">tstat</a>, <a href="https://www.rfc-editor.org/info/rfc3176" target="_blank" rel="noopener noreferrer">sflow</a>, or <a href="https://www.cisco.com/c/en/us/products/collateral/ios-nx-os-software/ios-netflow/prod_white_paper0900aecd80406232.html" target="_blank" rel="noopener noreferrer">netflow</a>) and send it to a &quot;pipeline host&quot; for processing. </p><p>Tstat flow data can be sent directly to the pipeline ingest RabbitMQ queue on the pipeline host using the Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> tool. This can be installed as usual or via Docker. </p><p>Sflow and netflow data from configured routers should be sent to the pipeline host where it is collected and stored into nfcapd files using <a href="https://github.com/phaag/nfdump" target="_blank" rel="noopener noreferrer">nfdump tools</a>. The Netsage project has packaged the nfdump tools into a <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">Docker container</a> for ease of use.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-components"></a>Pipeline Components<a class="hash-link" href="#pipeline-components" title="Direct link to heading">#</a></h2><p>The Netsage Flow Processing Pipeline is made of the following components</p><ul><li>Importer:  Perl scripts on the pipeline host that read nfcapd flow files and send the flow data to a RabbitMQ queue.   (<a href="/netsage-pipeline/docs/pipeline/importer">Doc</a>, <a href="https://github.com/netsage-project/netsage-pipeline/blob/master/lib/GRNOC/NetSage/Deidentifier/NetflowImporter.pm" target="_blank" rel="noopener noreferrer">in github</a>)</li><li><a href="https://www.rabbitmq.com/" target="_blank" rel="noopener noreferrer">RabbitMQ</a>: Used for message passing and queuing of tasks.</li><li><a href="https://www.elastic.co/logstash" target="_blank" rel="noopener noreferrer">Logstash</a> pipeline: Performs a variety of operations on the flow data to transform it and add additional information.  (<a href="/netsage-pipeline/docs/pipeline/logstash">Doc</a>)</li><li><a href="https://www.elastic.co/what-is/elasticsearch" target="_blank" rel="noopener noreferrer">Elasticsearch</a>: Used for storing the final flow data. </li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="visualization"></a>Visualization<a class="hash-link" href="#visualization" title="Direct link to heading">#</a></h2><p><a href="https://grafana.com/oss/grafana/" target="_blank" rel="noopener noreferrer">Grafana</a> or <a href="https://www.elastic.co/kibana" target="_blank" rel="noopener noreferrer">Kibana</a> can be used to visualize the data stored in elasticsearch.  Netsage Grafana Dashboards are available <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">in github</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="pipeline-installation"></a>Pipeline Installation<a class="hash-link" href="#pipeline-installation" title="Direct link to heading">#</a></h2><p>Originally, the pipeline was deployed by installing all of the components individually on one or more servers (the &quot;BareMetal&quot; or &quot;Manual&quot; Install). More recently, we&#x27;ve also added a Docker deployment option. With simple pipelines having just one sflow and/or one netflow sensor (and any number of tstat sensors), the basic &quot;Docker Installation&quot; should suffice. The &quot;Docker Advanced Options&quot; guide will help when there are more sensors and/or other customizations required.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/intro.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/tstat"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tstat Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#description" class="table-of-contents__link">Description</a></li><li><a href="#data-collection" class="table-of-contents__link">Data Collection</a></li><li><a href="#pipeline-components" class="table-of-contents__link">Pipeline Components</a></li><li><a href="#visualization" class="table-of-contents__link">Visualization</a></li><li><a href="#pipeline-installation" class="table-of-contents__link">Pipeline Installation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f2cce206.f146a973.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/logstash/index.html b/docs/pipeline/logstash/index.html
index 7e04bf05..29e8728d 100644
--- a/docs/pipeline/logstash/index.html
+++ b/docs/pipeline/logstash/index.html
@@ -5,21 +5,21 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9e1ec80c.ab114838.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Logstash Pipeline | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Logstash Pipeline | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><meta data-react-helmet="true" property="og:description" content="The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f0b0282f.243b745a.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/logstash">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/logstash">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/logstash">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/logstash">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/logstash">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/logstash">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/logstash">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/logstash">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Logstash Pipeline</h1></header><div class="markdown"><p>The Logstash portion of the Netsage Pipeline reads in flows from a RabbitMQ queue, performs various transformations and adds additional information to them, then sends them to a location specified in the output logstash config, eventually ending up in an Elasticsearch instance. </p><p>Logstash config files invoke various logstash &quot;filters&quot; and actions. These conf files are located in /etc/logstash/conf.d/. See below for a brief description of what each does and check the files for comments.</p><p>Notes: </p><ul><li>All *.conf files in conf.d/ are executed in alphabetical order, as if they were one huge file. Those ending in .disabled will not be executed (assuming &#x27;path.config: &quot;/etc/logstash/conf.d/*.conf&quot;&#x27; in /etc/logstash/pipelines.yml).</li><li>If actions in a particular .conf file are not needed in your particular case, they can be removed or the file disabled, but check carefully for effects on downstream configs.</li><li>MaxMind, CAIDA, and Science Registry database files required by the geoip and aggregate filters are downloaded from scienceregistry.netsage.global via cron jobs weekly or daily. (MaxMind data can change weekly, CAIDA quarterly, Science Registry information randomly.) <strong>NOTE that new versions won&#x27;t be used in the pipeline until logstash is restarted.</strong> There is a cron file to do this also, though it&#x27;s not running in Docker deployments. Similarly for other support files, eg, those used in 90-additional-fields.conf.</li><li>Lookup tables for 55-member-orgs.conf that we have compiled are available from sciencregistry.grnoc.iu.edu. See the cron files provided. These will not be updated often, so you may run the cron jobs or not. You will need to provide lists for other networks yourself or ask us.</li></ul><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="logstash-sequence"></a>Logstash Sequence<a class="hash-link" href="#logstash-sequence" title="Direct link to heading">#</a></h2><p>The main things done in each conf file are as follows.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="01-input-rabbitconf"></a>01-input-rabbit.conf<a class="hash-link" href="#01-input-rabbitconf" title="Direct link to heading">#</a></h3><p>Reads flows from a rabbitmq queue. (The &quot;.disabled&quot; extenstion can be removed from other 01-input configs available in conf.d/ to get flows from other sources.)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="10-preliminariesconf"></a>10-preliminaries.conf<a class="hash-link" href="#10-preliminariesconf" title="Direct link to heading">#</a></h3><p>Drops flows to or from private IP addresses;
 converts any timestamps in milliseconds to seconds;
 drops events with timestamps more than a year in the past or (10 sec) in the future;
 sets duration and rates to 0 if duration is &lt;= 0.002 sec (because tiny durations/few samples lead to inaccurate rates)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="15-sensor-specific-changesconf"></a>15-sensor-specific-changes.conf<a class="hash-link" href="#15-sensor-specific-changesconf" title="Direct link to heading">#</a></h3><p>Makes any changes to fields needed for specific sensors. This config currently provides 1) the ability to drop all flows that do not use interfaces (ifindexes) in a specfied list; lists can be sensor-specific, 2) the ability to change the sensor name for flows from a specified sensor which use a certain interface, and 3) the ability to apply a sampling rate correction manually for named sensors. You may edit the file in a bare-metal installation and specify everything explicitly (upgrades will not overwrite this config) or you may use the environment file specified in the systemd unit file. For Docker installations, use the .env file to specifiy the parameters. By default, this config will do nothing since the flags will be set to False.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="20-add_idconf"></a>20-add_id.conf<a class="hash-link" href="#20-add_idconf" title="Direct link to heading">#</a></h3><p>Adds a unique id (evenutally called meta.id) which is a hash of the 5-tuple of the flow (src and dst ips and ports, and protocol) plus the sensor name. This id is used for aggregating (stitching) in the next step. </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="40-aggregationconf"></a>40-aggregation.conf<a class="hash-link" href="#40-aggregationconf" title="Direct link to heading">#</a></h3><p>Stitches together flows from different nfcapd files into longer flows, matching them up by meta.id and using a specified inactivity_timeout to decide when to start a new flow.</p><p>Notes: </p><ul><li>By default, 5-minute nfcapd files are assumed and the inactivity_timeout is set to 10.5 minutes. If more than 10.5 min have passed between the start of the current flow and the start of the last matching one, do not stitch them together.</li><li>If your nfcapd files are written every 15 minutes, change the inactivity_timeout to at least 16 minutes.</li><li>There is another &quot;timeout&quot; setting which is basically the maximum duration of a stitched flow (default: 24 hr).</li><li>When logstash shuts down, any flows &quot;in the aggregator&quot; will be written out to aggregate_maps_path (default: /tmp/logstash-aggregation-maps). The file is then read back in when logstash is restarted so aggregation can continue. </li><li>Your logstash pipeline can have only 1 worker or aggregation is not going to work! This is set in the logstash config file.</li><li>Tstat flows come in already complete, so no aggregation is done on those flows.</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="45-geoip-taggingconf"></a>45-geoip-tagging.conf<a class="hash-link" href="#45-geoip-taggingconf" title="Direct link to heading">#</a></h3><p>Queries the MaxMind GeoLite2-City database by IP to get src and dst Countries, Continents, Latitudes, and Longitudes;
@@ -27,16 +27,16 @@
 Sets ASN to -1 if it is unavailable for any reason.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="53-caida-orgconf"></a>53-caida-org.conf<a class="hash-link" href="#53-caida-orgconf" title="Direct link to heading">#</a></h3><p>Uses the current source and destination ASNs to get organization names from the prepared CAIDA ASN-to-Organization lookup file.</p><p><em>This product uses a lookup table constructed from the CAIDA AS Organizations Dataset - see <a href="http://www.caida.org/data/as-organizations" target="_blank" rel="noopener noreferrer">www.caida.org</a>.</em> </p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="55-member-orgsconf"></a>55-member-orgs.conf<a class="hash-link" href="#55-member-orgsconf" title="Direct link to heading">#</a></h3><p>Searches any provided lookup tables by IP to obtain member or customer organization names and overwrite the Organization determined previously.
 This allows entities which don&#x27;t own their own ASs to be listed as the src or dst Organization.</p><p>Note: These lookup tables are not stored in github, but an example is provided to show the layout and tables we have can be downloaded via a cron job.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="60-scireg-tagging-fakegeoipconf"></a>60-scireg-tagging-fakegeoip.conf<a class="hash-link" href="#60-scireg-tagging-fakegeoipconf" title="Direct link to heading">#</a></h3><p>Uses a fake geoip database containing <a href="http://scienceregistry.grnoc.iu.edu" target="_blank" rel="noopener noreferrer">Science Registry</a> information to tag the flows with source and destination science disciplines and roles, organizations and locations, etc;
 removes Registry fields we don&#x27;t need to save to elasticsearch.</p><p>Notes: </p><ul><li>The <a href="https://scienceregistry.netsage.global/rdb/" target="_blank" rel="noopener noreferrer">Science Registry</a> stores human-curated information about various &quot;resources&quot;. Resources are sources and destinations of flows.</li><li>The Science Registry &quot;fake geoip database&quot; is updated weekly and can be downloaded via wget in a cron job (provided in the installation).</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="70-deidentifyconf"></a>70-deidentify.conf<a class="hash-link" href="#70-deidentifyconf" title="Direct link to heading">#</a></h3><p>Replaces the last octet of IPv4 addresses and the last 4 hextets of IPv6 addresses with x&#x27;s in order to deidentify them.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="80-privatizeorgconf"></a>80-privatize.org.conf<a class="hash-link" href="#80-privatizeorgconf" title="Direct link to heading">#</a></h3><p>Removes information about Australian organizations (or, with modification, any country that has privacy rules that require us not to identify organizations).
-If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/9e1ec80c.ab114838.js"></script>
+If the ASN is one of those listed, completely replaces the IP with x&#x27;s, sets the location to central Autralia, sets all organizations to &quot;AARNet&quot;, removes all Projects.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="88-preferred-location-orgconf"></a>88-preferred-location-org.conf<a class="hash-link" href="#88-preferred-location-orgconf" title="Direct link to heading">#</a></h3><p>Copies Science Registry organization and location values, if they exist, to the meta.preferred_organization and meta.preferred_location fields. If there are no Science Registry values, the organizations and locations from the CAIDA and MaxMind lookups, respectively, are saved to those fields.</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="90-additional-fieldsconf"></a>90-additional-fields.conf<a class="hash-link" href="#90-additional-fieldsconf" title="Direct link to heading">#</a></h3><p>Sets additional quick and easy fields.  Supporting mapping or ruby files are used - see support/ and ruby/ in conf.d/. Currently we have (for Netsage&#x27;s use):</p><ul><li>sensor_group  = TACC, AMPATH, etc.  (based on matching sensor names to regexes)</li><li>sensor_type   = Circuit, Archive, Exchange Point, or Regional Network  (based on matching sensor names to regexes)</li><li>country_scope = Domestic, International, or Mixed  (based on src and dst countries and possibly continents, where Domestic = US, Puerto Rico, or Guam)</li><li>is_network_testing = yes, no  (yes if discipline from the science registry is &#x27;CS.Network Testing and Monitoring&#x27; or port = 5001, 5101, or 5201)</li><li>es_doc_id = hash of meta.id and the start time of the flow. If this id is used as the document id in elasticsearch, flows that are mistakenly input more than once will update existing documents rather than be added as duplicates. (NOTE: due to how netflow works, use es_doc_id as the ES document id only for sflow!)</li></ul><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="95-cleanupconf"></a>95-cleanup.conf<a class="hash-link" href="#95-cleanupconf" title="Direct link to heading">#</a></h3><p>Does small misc. tasks at the end like rename, remove, or convert fields</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="98-post-processconf"></a>98-post-process.conf<a class="hash-link" href="#98-post-processconf" title="Direct link to heading">#</a></h3><p>Adds @exit_time and @processing_time (these are mainly for developers)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="99-output-rabbitconf"></a>99-output-rabbit.conf<a class="hash-link" href="#99-output-rabbitconf" title="Direct link to heading">#</a></h3><p>Sends results to a final RabbitMQ queue. (&quot;.disabled&quot; can be removed from other output configs to send flows to other places)</p><h3><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="final-stage"></a>Final Stage<a class="hash-link" href="#final-stage" title="Direct link to heading">#</a></h3><p>In the GlobalNOC-Netsage case, the output filter writes the flows to a network-specific RabbitMQ queue on another host and the last stage is a separate logstash pipeline on a 3rd host. The latter reads flows from the final queue using a rabbitmq input filter and sends it into elasticsearch using an elasticsearch output filter with a mapping template which sets data types for the fields. </p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="field-names"></a>Field names<a class="hash-link" href="#field-names" title="Direct link to heading">#</a></h2><p>The fields used/created in Logstash (and saved to Elasticsearch) are listed in the <a href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch doc</a>.</p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/logstash.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/importer"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Importer</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/elastic"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Elasticsearch »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#logstash-sequence" class="table-of-contents__link">Logstash Sequence</a><ul><li><a href="#01-input-rabbitconf" class="table-of-contents__link">01-input-rabbit.conf</a></li><li><a href="#10-preliminariesconf" class="table-of-contents__link">10-preliminaries.conf</a></li><li><a href="#15-sensor-specific-changesconf" class="table-of-contents__link">15-sensor-specific-changes.conf</a></li><li><a href="#20-add_idconf" class="table-of-contents__link">20-add_id.conf</a></li><li><a href="#40-aggregationconf" class="table-of-contents__link">40-aggregation.conf</a></li><li><a href="#45-geoip-taggingconf" class="table-of-contents__link">45-geoip-tagging.conf</a></li><li><a href="#50-asnconf" class="table-of-contents__link">50-asn.conf</a></li><li><a href="#53-caida-orgconf" class="table-of-contents__link">53-caida-org.conf</a></li><li><a href="#55-member-orgsconf" class="table-of-contents__link">55-member-orgs.conf</a></li><li><a href="#60-scireg-tagging-fakegeoipconf" class="table-of-contents__link">60-scireg-tagging-fakegeoip.conf</a></li><li><a href="#70-deidentifyconf" class="table-of-contents__link">70-deidentify.conf</a></li><li><a href="#80-privatizeorgconf" class="table-of-contents__link">80-privatize.org.conf</a></li><li><a href="#88-preferred-location-orgconf" class="table-of-contents__link">88-preferred-location-org.conf</a></li><li><a href="#90-additional-fieldsconf" class="table-of-contents__link">90-additional-fields.conf</a></li><li><a href="#95-cleanupconf" class="table-of-contents__link">95-cleanup.conf</a></li><li><a href="#98-post-processconf" class="table-of-contents__link">98-post-process.conf</a></li><li><a href="#99-output-rabbitconf" class="table-of-contents__link">99-output-rabbit.conf</a></li><li><a href="#final-stage" class="table-of-contents__link">Final Stage</a></li></ul></li><li><a href="#field-names" class="table-of-contents__link">Field names</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f0b0282f.243b745a.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/nfdump/index.html b/docs/pipeline/nfdump/index.html
index 10fcb5cc..57c27289 100644
--- a/docs/pipeline/nfdump/index.html
+++ b/docs/pipeline/nfdump/index.html
@@ -5,30 +5,30 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Sflow/Netflow Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Sflow/Netflow Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/59305d05.7e6bc501.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Sflow/Netflow Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Sflow/Netflow Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><meta data-react-helmet="true" property="og:description" content="Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6."><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/092c2d1d.97793caf.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/nfdump">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/59305d05.7e6bc501.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/nfdump">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/nfdump">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/nfdump">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/nfdump">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/nfdump">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/nfdump">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/nfdump">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Sflow/Netflow Data Collection</h1></header><div class="markdown"><p>Sflow and Netflow export can be configured on appropriate network devices. Netsage uses tools in the Nfdump package to collect and process the resulting flow data.  The toolset supports netflow v1, v5/v7, v9, IPFIX and SFLOW, IPv4 as well as IPv6.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-usage"></a>Netsage Usage<a class="hash-link" href="#netsage-usage" title="Direct link to heading">#</a></h2><p>Nfcapd and/or sfcapd processes (from the nfdump package) are used to collect incoming netflow and/or sflow data and save it to disk in nfcapd files.  The files are then read by the <a href="/netsage-pipeline/docs/pipeline/importer">importer</a>, which uses an nfdump command, and sent to RabbitMQ. From there, the <a href="/netsage-pipeline/docs/pipeline/logstash">logstash</a> pipeline ingests the flows and processes them in exactly the same way as it processes tstat flows.  The data is eventually saved in elasticsearch and visualized by <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">grafana dashboards</a>.</p><p>One may also use the nfdump command interactively to view the flows in a nfcapd file in a terminal window.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker-deployment"></a>Docker Deployment<a class="hash-link" href="#docker-deployment" title="Direct link to heading">#</a></h2><p>The nfdump/nfcapd/sfcapd processes can be invoked locally or using a Docker container.  The Docker deployment of the Pipeline uses an nfdump Docker container. (See the Docker Deployment Guide.) The Docker image definitions can be found <a href="https://github.com/netsage-project/docker-nfdump-collector" target="_blank" rel="noopener noreferrer">HERE</a></p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/nfdump.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/tstat"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Tstat Data Collection</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/importer"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Importer »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-usage" class="table-of-contents__link">Netsage Usage</a></li><li><a href="#docker-deployment" class="table-of-contents__link">Docker Deployment</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/092c2d1d.97793caf.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/pipeline/tstat/index.html b/docs/pipeline/tstat/index.html
index b4c1d2d8..ede2d62e 100644
--- a/docs/pipeline/tstat/index.html
+++ b/docs/pipeline/tstat/index.html
@@ -5,30 +5,30 @@
 <meta name="viewport" content="width=device-width,initial-scale=1">
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
-<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.0.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.0.0"><meta data-react-helmet="true" property="og:title" content="Tstat Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage GitHub Project"><meta data-react-helmet="true" property="og:description" content="Netsage GitHub Project"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/161.d19b2b8e.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/162.71673972.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/9aa54785.d3836c06.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/17896441.1b171940.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/8084fbd4.5592d29d.js" as="script">
+<link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Tstat Data Collection | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_version" content="2.1.0"><meta data-react-helmet="true" name="docusaurus_tag" content="docs-default-2.1.0"><meta data-react-helmet="true" property="og:title" content="Tstat Data Collection | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Netsage GitHub Project"><meta data-react-helmet="true" property="og:description" content="Netsage GitHub Project"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat" hreflang="x-default"><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/157.1f338186.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/158.319f81c0.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/63503174.24f9a6f6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/17896441.104e0abb.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/f9995f72.b29de3ff.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/tstat">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/choose_install">Choose Install</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/bare_metal_install">Manual Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.0.0</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.0.0/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/161.d19b2b8e.js"></script>
-<script src="/netsage-pipeline/assets/js/162.71673972.js"></script>
-<script src="/netsage-pipeline/assets/js/9aa54785.d3836c06.js"></script>
-<script src="/netsage-pipeline/assets/js/17896441.1b171940.js"></script>
-<script src="/netsage-pipeline/assets/js/8084fbd4.5592d29d.js"></script>
+<div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a aria-current="page" class="menu__link navbar__link--active" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/tstat">master (unreleased)</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active" href="/netsage-pipeline/docs/pipeline/tstat">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/tstat">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/tstat">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/tstat">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/tstat">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/tstat">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper main-docs-wrapper"><div class="docPage_31aa"><div class="docSidebarContainer_3Kbt" role="complementary"><div class="sidebar_15mo"><div class="menu menu--responsive thin-scrollbar menu_Bmed"><button aria-label="Open menu" aria-haspopup="true" class="button button--secondary button--sm menu__button" type="button"><svg aria-label="Menu" class="sidebarMenuIcon_fgN0" width="24" height="24" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><ul class="menu__list"><li class="menu__list-item"><a class="menu__link menu__link--sublist menu__link--active" href="#!">Pipeline</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/intro">Intro</a></li><li class="menu__list-item"><a aria-current="page" class="menu__link menu__link--active active" tabindex="0" href="/netsage-pipeline/docs/pipeline/tstat">Tstat Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/nfdump">Sflow/Netflow Data</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/importer">Importer</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/logstash">Logstash</a></li><li class="menu__list-item"><a class="menu__link" tabindex="0" href="/netsage-pipeline/docs/pipeline/elastic">Elasticsearch</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Deployment</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_simple">Docker Installation</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_install_advanced">Docker Advanced Options</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_upgrade">Docker - Upgrading</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/deploy/docker_troubleshoot">Troubleshooting</a></li></ul></li><li class="menu__list-item menu__list-item--collapsed"><a class="menu__link menu__link--sublist" href="#!">Development</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/dev_dataset">Replay Dataset</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_guide">Docker Dev Guide</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docusaurus">Docusaurus</a></li><li class="menu__list-item"><a class="menu__link" tabindex="-1" href="/netsage-pipeline/docs/devel/docker_dev_tag">Making Releases</a></li></ul></li></ul></div></div></div><main class="docMainContainer_3ufF"><div class="container padding-vert--lg docItemWrapper_3FMP"><div class="row"><div class="col docItemCol_3FnS"><div class="docItemContainer_33ec"><article><div><span class="badge badge--secondary">Version: 2.1.0</span></div><header><h1 class="docTitle_3a4h">Tstat Data Collection</h1></header><div class="markdown"><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="netsage-github-project"></a>Netsage GitHub Project<a class="hash-link" href="#netsage-github-project" title="Direct link to heading">#</a></h2><p><a href="http://tstat.polito.it/" target="_blank" rel="noopener noreferrer">Tstat</a> is a passive sniffer that provides insights into traffic patterns.  The Netsage <a href="https://github.com/netsage-project/tstat-transport" target="_blank" rel="noopener noreferrer">tstat-transport</a> project provides client programs to parse the captured data and send it to a rabbitmq host where it can then be processed by the <a href="/netsage-pipeline/docs/pipeline/logstash">logstash pipeline</a>, stored in elasticsearch, and finally displayed in our Grafana <a href="https://github.com/netsage-project/netsage-grafana-configs" target="_blank" rel="noopener noreferrer">dashboards</a>.</p><h2><a aria-hidden="true" tabindex="-1" class="anchor enhancedAnchor_2LWZ" id="docker"></a>Docker<a class="hash-link" href="#docker" title="Direct link to heading">#</a></h2><p>Netsage Docker images exist on Docker Hub for tstat and tstat_transport. This is still in a beta state and is in development.  The initial documentation is available <a href="https://github.com/netsage-project/tstat-transport/blob/master/docs/docker.md" target="_blank" rel="noopener noreferrer">here</a>.  </p></div></article><div class="margin-vert--xl"><div class="row"><div class="col"><a href="https://github.com/netsage-project/netsage-pipeline/edit/master/website/versioned_docs/version-2.1.0/pipeline/tstat.md" target="_blank" rel="noreferrer noopener"><svg fill="currentColor" height="1.2em" width="1.2em" preserveAspectRatio="xMidYMid meet" role="img" viewBox="0 0 40 40" class="iconEdit_2_ui" aria-label="Edit page"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></div></div><div class="margin-vert--lg"><nav class="pagination-nav" aria-label="Docs pages navigation"><div class="pagination-nav__item"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/intro"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">« Intro</div></a></div><div class="pagination-nav__item pagination-nav__item--next"><a class="pagination-nav__link" href="/netsage-pipeline/docs/pipeline/nfdump"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Sflow/Netflow Data Collection »</div></a></div></nav></div></div></div><div class="col col--3"><div class="tableOfContents_35-E thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#netsage-github-project" class="table-of-contents__link">Netsage GitHub Project</a></li><li><a href="#docker" class="table-of-contents__link">Docker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/157.1f338186.js"></script>
+<script src="/netsage-pipeline/assets/js/158.319f81c0.js"></script>
+<script src="/netsage-pipeline/assets/js/63503174.24f9a6f6.js"></script>
+<script src="/netsage-pipeline/assets/js/17896441.104e0abb.js"></script>
+<script src="/netsage-pipeline/assets/js/f9995f72.b29de3ff.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/img/favicon.ico:Zone.Identifier b/img/favicon.ico:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/favicon.ico:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/logo.png:Zone.Identifier b/img/logo.png:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/logo.png:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/logo.svg:Zone.Identifier b/img/logo.svg:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/logo.svg:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/netsage.webp:Zone.Identifier b/img/netsage.webp:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/netsage.webp:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/netsage_bg.webp:Zone.Identifier b/img/netsage_bg.webp:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/netsage_bg.webp:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/undraw_docusaurus_mountain.svg:Zone.Identifier b/img/undraw_docusaurus_mountain.svg:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/undraw_docusaurus_mountain.svg:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/undraw_docusaurus_react.svg:Zone.Identifier b/img/undraw_docusaurus_react.svg:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/undraw_docusaurus_react.svg:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/img/undraw_docusaurus_tree.svg:Zone.Identifier b/img/undraw_docusaurus_tree.svg:Zone.Identifier
deleted file mode 100644
index c6c0ff29..00000000
--- a/img/undraw_docusaurus_tree.svg:Zone.Identifier
+++ /dev/null
@@ -1,3 +0,0 @@
-[ZoneTransfer]
-ZoneId=3
-ReferrerUrl=C:\Users\Doug Southworth\Documents\Work\NetSage Configs\netsage-pipeline-1.2.12.zip
diff --git a/index.html b/index.html
index 9be5c8cb..83c95161 100644
--- a/index.html
+++ b/index.html
@@ -6,21 +6,21 @@
 <meta name="generator" content="Docusaurus v2.0.0-alpha.72">
 <link rel="alternate" type="application/rss+xml" href="/netsage-pipeline/blog/rss.xml" title="Netsage Pipeline Documentation Blog RSS Feed">
 <link rel="alternate" type="application/atom+xml" href="/netsage-pipeline/blog/atom.xml" title="Netsage Pipeline Documentation Blog Atom Feed"><title data-react-helmet="true">Netsage Pipeline Documentation | Netsage Pipeline Documentation</title><meta data-react-helmet="true" property="og:title" content="Netsage Pipeline Documentation | Netsage Pipeline Documentation"><meta data-react-helmet="true" name="description" content="Description will go into a meta tag in &lt;head /&gt;"><meta data-react-helmet="true" property="og:description" content="Description will go into a meta tag in &lt;head /&gt;"><meta data-react-helmet="true" property="og:url" content="https://netsage-project.github.io/netsage-pipeline/"><meta data-react-helmet="true" name="docusaurus_locale" content="en"><meta data-react-helmet="true" name="docusaurus_tag" content="default"><link data-react-helmet="true" rel="shortcut icon" href="/netsage-pipeline/img/favicon.ico"><link data-react-helmet="true" rel="canonical" href="https://netsage-project.github.io/netsage-pipeline/"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/" hreflang="en"><link data-react-helmet="true" rel="alternate" href="https://netsage-project.github.io/netsage-pipeline/" hreflang="x-default"><script data-react-helmet="true">function maybeInsertBanner(){window.__DOCUSAURUS_INSERT_BASEURL_BANNER&&insertBanner()}function insertBanner(){var n=document.getElementById("docusaurus-base-url-issue-banner-container");if(n){n.innerHTML='\n<div id="docusaurus-base-url-issue-banner" style="border: thick solid red; background-color: rgb(255, 230, 179); margin: 20px; padding: 20px; font-size: 20px;">\n   <p style="font-weight: bold; font-size: 30px;">Your Docusaurus site did not load properly.</p>\n   <p>A very common reason is a wrong site <a href="https://v2.docusaurus.io/docs/docusaurus.config.js/#baseurl" style="font-weight: bold;">baseUrl configuration</a>.</p>\n   <p>Current configured baseUrl = <span style="font-weight: bold; color: red;">/netsage-pipeline/</span> </p>\n   <p>We suggest trying baseUrl = <span id="docusaurus-base-url-issue-banner-suggestion-container" style="font-weight: bold; color: green;"></span></p>\n</div>\n';var e=document.getElementById("docusaurus-base-url-issue-banner-suggestion-container"),s=window.location.pathname,r="/"===s.substr(-1)?s:s+"/";e.innerHTML=r}}window.__DOCUSAURUS_INSERT_BASEURL_BANNER=!0,document.addEventListener("DOMContentLoaded",maybeInsertBanner)</script><link rel="stylesheet" href="/netsage-pipeline/assets/css/styles.414b8674.css">
-<link rel="preload" href="/netsage-pipeline/assets/js/styles.19dac7a3.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.6b5de359.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/main.e20b6a1c.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/1.bb3bc4d5.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/2.89c52fba.js" as="script">
-<link rel="preload" href="/netsage-pipeline/assets/js/c4f5d8e4.76411a79.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/styles.ed63c98d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/runtime~main.09475183.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/main.16ba1fad.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/1.03db48d6.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/2.f7e8da3d.js" as="script">
+<link rel="preload" href="/netsage-pipeline/assets/js/c4f5d8e4.4e0e3f0d.js" as="script">
 </head>
 <body>
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
-<div id="docusaurus-base-url-issue-banner-container"></div><div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.0.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper"><header class="hero hero--primary heroBanner_3P7f"><div class="container"><h1 class="hero__title">Netsage Pipeline Documentation</h1><img src="img/netsage.webp"><p class="hero__subtitle">Netsage Pipeline Documentation</p><div class="buttons_1r9m"><a class="button button--outline button--secondary button--lg" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></div></div></header></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 NetSage Built with Docusaurus.</div></div></div></footer></div>
-<script src="/netsage-pipeline/assets/js/styles.19dac7a3.js"></script>
-<script src="/netsage-pipeline/assets/js/runtime~main.6b5de359.js"></script>
-<script src="/netsage-pipeline/assets/js/main.e20b6a1c.js"></script>
-<script src="/netsage-pipeline/assets/js/1.bb3bc4d5.js"></script>
-<script src="/netsage-pipeline/assets/js/2.89c52fba.js"></script>
-<script src="/netsage-pipeline/assets/js/c4f5d8e4.76411a79.js"></script>
+<div id="docusaurus-base-url-issue-banner-container"></div><div><a href="#main" class="skipToContent_1oUP">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Navigation bar toggle" class="navbar__toggle" type="button" tabindex="0"><svg aria-label="Menu" width="30" height="30" viewBox="0 0 30 30" role="img" focusable="false"><title>Menu</title><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="navbar__item navbar__link">Dashboard Documentation</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__item navbar__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li><a class="dropdown__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></div><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub</a><div class="react-toggle react-toggle--disabled displayOnlyInLargeViewport_GrZ2"><div class="react-toggle-track"><div class="react-toggle-track-check"><span class="toggle_71bT">🌜</span></div><div class="react-toggle-track-x"><span class="toggle_71bT">🌞</span></div></div><div class="react-toggle-thumb"></div><input type="checkbox" disabled="" aria-label="Dark mode toggle" class="react-toggle-screenreader-only"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div><div class="navbar-sidebar"><div class="navbar-sidebar__brand"><a class="navbar__brand" href="/netsage-pipeline/"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--light_3UqQ navbar__logo"><img src="/netsage-pipeline/img/logo.png" alt="NetSage Logo" class="themedImage_1VuW themedImage--dark_hz6m navbar__logo"><strong class="navbar__title">NetSage Documentation</strong></a></div><div class="navbar-sidebar__items"><div class="menu"><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></li><li class="menu__list-item"><a href="https://netsage-project.github.io/netsage-grafana-configs/" target="_self" rel="noopener noreferrer" class="menu__link">Dashboard Documentation</a></li><li class="menu__list-item menu__list-item--collapsed"><a role="button" class="menu__link menu__link--sublist">Versions</a><ul class="menu__list"><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/next/pipeline/intro">master (unreleased)</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/pipeline/intro">2.1.0</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.11/pipeline/intro">1.2.11</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.10/pipeline/intro">1.2.10</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.9/pipeline/intro">1.2.9</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.8/pipeline/intro">1.2.8</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.7/pipeline/intro">1.2.7</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.6/pipeline">1.2.6</a></li><li class="menu__list-item"><a class="menu__link" href="/netsage-pipeline/docs/1.2.5/pipeline">1.2.5</a></li></ul></li><li class="menu__list-item"><a href="https://github.com/netsage-project/netsage-pipeline/" target="_blank" rel="noopener noreferrer" class="menu__link">GitHub</a></li></ul></div></div></div></nav><div class="main-wrapper"><header class="hero hero--primary heroBanner_3P7f"><div class="container"><h1 class="hero__title">Netsage Pipeline Documentation</h1><img src="img/netsage.webp"><p class="hero__subtitle">Netsage Pipeline Documentation</p><div class="buttons_1r9m"><a class="button button--outline button--secondary button--lg" href="/netsage-pipeline/docs/pipeline/intro">Pipeline Documentation</a></div></div></header></div><footer class="footer footer--dark"><div class="container"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 NetSage Built with Docusaurus.</div></div></div></footer></div>
+<script src="/netsage-pipeline/assets/js/styles.ed63c98d.js"></script>
+<script src="/netsage-pipeline/assets/js/runtime~main.09475183.js"></script>
+<script src="/netsage-pipeline/assets/js/main.16ba1fad.js"></script>
+<script src="/netsage-pipeline/assets/js/1.03db48d6.js"></script>
+<script src="/netsage-pipeline/assets/js/2.f7e8da3d.js"></script>
+<script src="/netsage-pipeline/assets/js/c4f5d8e4.4e0e3f0d.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 659c33a9..12df3562 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://netsage-project.github.io/netsage-pipeline/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url></urlset>
\ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://netsage-project.github.io/netsage-pipeline/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.10/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.11/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/docker_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/deploy/install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/devel/docker</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.5/pipeline_logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/docker_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/deploy/install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/devel/docker</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.6/pipeline_logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.7/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.8/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/components/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/bare_metal_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/choose_install</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/1.2.9/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/next/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_env</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_first_steps</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/components/docker_pipeline</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_advanced</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_install_simple</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_troubleshoot</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/deploy/docker_upgrade</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/dev_dataset</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_guide</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docker_dev_tag</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/devel/docusaurus</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/elastic</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/importer</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/intro</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/logstash</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/nfdump</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://netsage-project.github.io/netsage-pipeline/docs/pipeline/tstat</loc><changefreq>weekly</changefreq><priority>0.5</priority></url></urlset>
\ No newline at end of file